Haystack code - Add a table to document store

//MARK: - Document Picker

// From your project's capabilities, enable both the iCloud and the Key-Sharing.

@IBAction func pickDocumentPressed(_ sender: Any) {
       presentDocumentPicker()
    }

extension OffersReceivedViewController : UIDocumentPickerDelegate,UINavigationControllerDelegate {
    
    func presentDocumentPicker() {
        let sTypes = getSupportedTypes()
        let documentPickerController = UIDocumentPickerViewController(
            forOpeningContentTypes: sTypes)
        
        documentPickerController.delegate = self
        documentPickerController.allowsMultipleSelection = false
        SVProgressHUD.show()
        self.present(documentPickerController, animated: true) {
            SVProgressHUD.dismiss()
        }
    }
    
    public func documentPicker(_ controller: UIDocumentPickerViewController, didPickDocumentsAt urls: [URL]) {
        guard let myURL = urls.first else {
            return
        }
        print("import result : \(myURL)")
        print("get Image Url")
        fileUrl = myURL
    }
    
    func documentPickerWasCancelled(_ controller: UIDocumentPickerViewController) {
        print("view was cancelled")
        dismiss(animated: true, completion: nil)
    }
    
    func getSupportedTypes() -> [UTType] {
        let supportedTypes : [UTType] = [UTType.utf8TabSeparatedText, UTType.rtf,    UTType.pdf, UTType.webArchive, UTType.image, UTType.jpeg,    UTType.tiff, UTType.gif, UTType.png, UTType.bmp, UTType.ico,    UTType.rawImage, UTType.svg, UTType.livePhoto, UTType.movie,    UTType.video, UTType.audio, UTType.quickTimeMovie, UTType.mpeg,    UTType.mpeg2Video, UTType.mpeg2TransportStream, UTType.mp3,    UTType.mpeg4Movie, UTType.mpeg4Audio, UTType.avi, UTType.aiff,    UTType.wav, UTType.midi, UTType.archive, UTType.gzip, UTType.bz2,    UTType.zip, UTType.appleArchive, UTType.spreadsheet, UTType.epub]
        return supportedTypes
    }
}

find . -name '*.wav' -exec ffmpeg -i '{}' -ar 16000 -ac 1 '{}.mono.wav' \;

find . -name '*.wav' -exec ffmpeg -i '{}' -filter:a "highpass=f=1200" '{}.phone.wav' \;

find . -name '*.wav' -exec ffmpeg -i '{}' -i city-traffic-outdoor.mp3 -filter_complex "[0:a][1:a]amerge=inputs=2[a]" -map "[a]" -ar 16000 -ac 1 '{}.city_traffic_outdoor.wav' \;

from IPython.display import YouTubeVideo


YOUTUBE_ID = 'xxxxxxxxxxxx'
YouTubeVideo(YOUTUBE_ID)

!rm -rf *.wav
!youtube-dl --extract-audio --audio-format wav --output "downloaded.%(ext)s" https://www.youtube.com/watch\?v\={YOUTUBE_ID}
!ffmpeg -loglevel panic -y -i downloaded.wav -acodec pcm_s16le -ac 1 -ar 16000 {project_name}/test.wav

# Add the tables to the DocumentStore

import json
from haystack import Document
import pandas as pd


def read_tables(filename):
    processed_tables = []
    with open(filename) as tables:
        tables = json.load(tables)
        for key, table in tables.items():
            current_columns = table["header"]
            current_rows = table["data"]
            current_df = pd.DataFrame(columns=current_columns, data=current_rows)
            document = Document(content=current_df, content_type="table", id=key)
            processed_tables.append(document)

    return processed_tables


tables = read_tables(f"{doc_dir}/tables.json")
document_store.write_documents(tables, index=document_index)

# Showing content field and meta field of one of the Documents of content_type 'table'
print(tables[0].content)
print(tables[0].meta)

//JavaScript
$(document).ready(function() {
  var playing = false;

  // Add file names.
  $('.audio-button').each(function() {
    var $button = $(this);    
    var $audio = $button.find('audio');
    
    $($('<span>').text($audio.attr('src'))).insertBefore($audio);
  });

  // Add click listener.
  $('.audio-button').click(function() {
    var $button = $(this);    
    var audio = $button.find('audio')[0]; // <-- Interact with this!
    
    // Toggle play/pause
    if (playing !== true) {
      audio.play();
    } else {
      audio.pause();
    }

    // Flip state
    $button.toggleClass('playing');
    playing = !playing
  });
});


// CSS
.fa.audio-button {
  position:      relative;
  display:       block;
  width:         12em;
  height:        2.25em;
  margin-bottom: 0.125em;
  text-align:    left;
}

.fa.audio-button:after {
  position:      absolute;
  right:         0.8em;
  content:       "\f04b";
}

.fa.audio-button.playing:after {
  content:"\f04c";
}

// HTML
<link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.6.3/css/font-awesome.min.css" rel="stylesheet"/>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>

<button class="fa audio-button">
  <audio src="media/test.mp3"></audio>
</button>
<button class="fa audio-button">
  <audio src="media/test2.mp3"></audio>
</button>
<button class="fa audio-button">
  <audio src="media/test3.mp3"></audio>
</button>
<button class="fa audio-button">
  <audio src="media/test4.mp3"></audio>
</button>

# Install the latest release of Haystack in your own environment
#! pip install farm-haystack

# Install the latest main of Haystack
!pip install --upgrade pip
!pip install git+https://github.com/deepset-ai/haystack.git#egg=farm-haystack[colab]

# Imports needed to run this notebook

from pprint import pprint
from tqdm import tqdm
from haystack.nodes import QuestionGenerator, BM25Retriever, FARMReader
from haystack.document_stores import ElasticsearchDocumentStore
from haystack.pipelines import (
    QuestionGenerationPipeline,
    RetrieverQuestionGenerationPipeline,
    QuestionAnswerGenerationPipeline,
)
from haystack.utils import launch_es, print_questions
  
 # Option 2: In Colab / No Docker environments: Start Elasticsearch from source
! wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.2-linux-x86_64.tar.gz -q
! tar -xzf elasticsearch-7.9.2-linux-x86_64.tar.gz
! chown -R daemon:daemon elasticsearch-7.9.2

import os
from subprocess import Popen, PIPE, STDOUT

es_server = Popen(
    ["elasticsearch-7.9.2/bin/elasticsearch"], stdout=PIPE, stderr=STDOUT, preexec_fn=lambda: os.setuid(1)  # as daemon
)
# wait until ES has started
! sleep 30

!wget https://dl.xpdfreader.com/xpdf-tools-linux-4.04.tar.gz
!tar -xvf xpdf-tools-linux-4.04.tar.gz && sudo cp xpdf-tools-linux-4.04/bin64/pdftotext /usr/local/bin

from haystack.nodes import TextConverter, PDFToTextConverter, DocxToTextConverter, PreProcessor


converter = TextConverter(remove_numeric_tables=True, valid_languages=["nl"])
doc_txt = converter.convert(file_path="/content/data/Chatbot_BVO DDK_22092022.txt", meta=None)[0]

from haystack.nodes import PreProcessor


# This is a default usage of the PreProcessor.
# Here, it performs cleaning of consecutive whitespaces
# and splits a single large document into smaller documents.
# Each document is up to 1000 words long and document breaks cannot fall in the middle of sentences
# Note how the single document passed into the document gets split into 5 smaller documents

preprocessor = PreProcessor(
    clean_empty_lines=True,
    clean_whitespace=True,
    clean_header_footer=False,
    split_by="word",
    split_length=100,
    split_respect_sentence_boundary=True,
    language="nl",
)
docs = preprocessor.process([doc_txt])
print(f"n_docs_input: 1\nn_docs_output: {len(docs)}")

# Initialize Question Generator
question_generator = QuestionGenerator()

# Fill the document store with a German document.
document_store.write_documents(docs)

# Load machine translation models
from haystack.nodes import TransformersTranslator
in_translator = TransformersTranslator(model_name_or_path="Helsinki-NLP/opus-mt-nl-en")
out_translator = TransformersTranslator(model_name_or_path="Helsinki-NLP/opus-mt-en-nl")

reader = FARMReader("deepset/roberta-base-squad2")
qag_pipeline = QuestionAnswerGenerationPipeline(question_generator, reader)

# Wrap the previously defined QuestionAnswerGenerationPipeline
from haystack.pipelines import TranslationWrapperPipeline

pipeline_with_translation = TranslationWrapperPipeline(
    input_translator=in_translator, output_translator=out_translator, pipeline=qag_pipeline
)

for idx, document in enumerate(tqdm(document_store)):
    print(f"\n * Generating questions and answers for document {idx}: {document.content[:100]}...\n")
    result = pipeline_with_translation.run(documents=[document])
    print_questions(result)

// ---------------------------------------------------------------------------
// Project Name		:	Nook
// File Name		:	Audio.c
// Author			:	Mary Khuu
// Creation Date	:	15 Mar 2021
// Purpose			:	add audio to the game
// All content © 2021 DigiPen (USA) Corporation, all rights reserved.
// ---------------------------------------------------------------------------

#include "fmod.h"
#include <stdio.h>		// printf()
#include <stdbool.h>	// FALSE
#include "AEEngine.h"

#include "Audio.h"

FMOD_SYSTEM* soundSystem;
FMOD_SOUND* sound;
FMOD_CHANNEL* channel;
FMOD_RESULT result;


// Initialize the Audio System
void AudioInit()
{
	channel = 0;

	// Create and Initialize the FMOD System
	result = FMOD_System_Create(&soundSystem);

	void* extradriverdata = 0;
	result = FMOD_System_Init(soundSystem, 32, FMOD_INIT_NORMAL, extradriverdata);
}

void playSound(bool trigger, const char* file)
{
	// Create and Play the sound
	// Note: this should be generalized for multiple sounds and
	//       be placed in a function to be used outside of init.
	result = FMOD_System_CreateStream(soundSystem, file, FMOD_LOOP_OFF | FMOD_2D, 0, &sound);

	result = FMOD_System_PlaySound(soundSystem, sound, 0, trigger, &channel);
	
}

void playSoundAdvanced(const char* file, float volume)
{
	FMOD_CHANNEL* channel;

	result = FMOD_System_CreateStream(soundSystem, file, FMOD_LOOP_OFF | FMOD_2D, 0, &sound);

	result = FMOD_System_PlaySound(soundSystem, sound, 0, true, &channel);

	result = FMOD_Channel_SetVolume(channel, volume);

	result = FMOD_Channel_SetPaused(channel, false);
}

// Update the Audio System
// Note: this should be called frequently such as every game loop or
//       every time a user enters a command depending on the engine
void AudioUpdate()
{
	result = FMOD_System_Update(soundSystem);
}

// Cleanup the Audio System
void AudioCleanup()
{
	// Release all sounds that have been created
	result = FMOD_Sound_Release(sound);

	// Close and Release the FMOD system
	result = FMOD_System_Close(soundSystem);
	result = FMOD_System_Release(soundSystem);
}

import pyaudio
import numpy as np
import pyqtgraph as pg

# Initialize PyAudio
pa = pyaudio.PyAudio()

# Set up audio stream
stream = pa.open(
    format=pyaudio.paInt16,
    channels=1,
    rate=44100,
    input=True,
    frames_per_buffer=1024
)

# Set up PyQTGraph
app = pg.mkQApp()
win = pg.GraphicsLayoutWidget()
win.show()
plot = win.addPlot(title='Real-time Audio Waveform')
curve = plot.plot()

# Function to update the plot
def update():
    wf_data = np.frombuffer(stream.read(1024), dtype=np.int16)
    curve.setData(wf_data)

# Start the audio stream
timer = pg.QtCore.QTimer()
timer.timeout.connect(update)
timer.start(50)

# Start Qt event loop
pg.mkQApp().exec()

import mido
import random

def generate_notes(notes_len):
    notes = []
    for i in range(notes_len):
        notes.append(random.randint(0, 127))

    return notes



def create_midi_file(notes):
    output_midi = mido.MidiFile()
    # create a track 
    track = mido.MidiTrack()
    # add track
    output_midi.tracks.append(track)

    # write notes
    for note in notes:
        track.append(mido.Message('note_on', note = note, velocity=64, time = 120))
        track.append(mido.Message('note_off', note = note, velocity=64, time = 120))

    # write midi file
    output_midi.save('out.mid')

notes = generate_notes(100)
create_midi_file(notes)

# You can use imageMagick to create a gradient image on the fly, but with all the weird possibilities I didn't quite get the hang of it (yet). Anyway, a default 50:50 gradient can be created via "convert -size 100x100 gradient:black-white target.png". To shift the center, you can chain multiple gradients and use the ""-append" flag
❯ convert -size 1000x600 gradient:black-black -size 1000x400 gradient:black-none -append gradient.png

# To use the gradient as a mask on your image, they should be the same size. You can easily resize the image. But keep in mind that imageMagixk respects the proportions and resizes X and Y according to the smaller value
❯ convert gradient.png -resize 1200x800 gradient_resized.png

# Here is how to combine the two, so that you're left with a new image with the mask applied
❯ convert content.jpg gradient_resized.png -alpha on -compose CopyOpacity -composite result.png

 const convertion = (value, last = 0) => {
    if (value > 0) {
      value = Math.round(value);
      var strlength = value.toString().length;
      var amount;
      if (strlength >= 4 && strlength < 6) {
        amount = Math.abs(value / 1000).toFixed(1) + "K";
      } else if (strlength >= 6 && strlength < 8) {
        if (last === 1) {
          amount = Math.abs(value / 100000).toFixed(1) + "+ Lakh";
        } else {
          amount = Math.abs(value / 100000).toFixed(1) + " Lakh";
        }
      } else if (strlength >= 8) {
        if (last == 1) {
          amount = Math.abs(value / 10000000).toFixed(1) + "+ Cr";
        } else {
          amount = Math.abs(value / 10000000).toFixed(1) + " Cr";
        }
      } else {
        amount = value;
      }
      return amount;
    }
  };

convert.currentData.experiments

convert.currentData.goals