Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bugfix/speech input on Safari/iOS (#1971) 

* debug to identify

* Safari sends audio file as mp4 and nor webp

* Safari on iOS needs special handling

* lint fixes

* updated condition

* Remove unused import

---------

Co-authored-by: Ilango <rajagopalilango@gmail.com>
This commit is contained in:
Vinod Kiran 2024-04-19 05:58:09 +05:30 committed by GitHub
parent d1c8f7eb96
commit 6bd8aaefc8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/components/src/speechToText.ts
View File

@ -10,7 +10,6 @@ export const convertSpeechToText = async (upload: IFileUpload, speechToTextConfi
const credentialId = speechToTextConfig.credentialId as string
const credentialData = await getCredentialData(credentialId ?? '', options)
const filePath = path.join(getStoragePath(), options.chatflowid, options.chatId, upload.name)
const audio_file = fs.createReadStream(filePath)
if (speechToTextConfig.name === 'openAIWhisper') {
@ -18,7 +17,6 @@ export const convertSpeechToText = async (upload: IFileUpload, speechToTextConfi
apiKey: credentialData.openAIApiKey
}
const openAIClient = new OpenAIClient(openAIClientOptions)
const transcription = await openAIClient.audio.transcriptions.create({
file: audio_file,
model: 'whisper-1',

3
packages/server/src/utils/buildChatflow.ts
View File

@ -66,6 +66,7 @@ export const utilBuildChatflow = async (req: Request, socketIO?: Server, isInter
fileUploads = incomingInput.uploads
for (let i = 0; i < fileUploads.length; i += 1) {
const upload = fileUploads[i]
if ((upload.type === 'file' || upload.type === 'audio') && upload.data) {
const filename = upload.name
const dir = path.join(getStoragePath(), chatflowid, chatId)
@ -83,7 +84,7 @@ export const utilBuildChatflow = async (req: Request, socketIO?: Server, isInter
}
// Run Speech to Text conversion
if (upload.mime === 'audio/webm') {
if (upload.mime === 'audio/webm' || upload.mime === 'audio/mp4') {
let speechToTextConfig: ICommonObject = {}
if (chatflow.speechToText) {
const speechToTextProviders = JSON.parse(chatflow.speechToText)

8
packages/ui/src/views/chatmessage/ChatMessage.jsx
View File

@ -241,7 +241,13 @@ export const ChatMessage = ({ open, chatflowid, isDialog, previews, setPreviews
}
const addRecordingToPreviews = (blob) => {
const mimeType = blob.type.substring(0, blob.type.indexOf(';'))
let mimeType = ''
const pos = blob.type.indexOf(';')
if (pos === -1) {
mimeType = blob.type
} else {
mimeType = blob.type.substring(0, pos)
}
// read blob and add to previews
const reader = new FileReader()
reader.readAsDataURL(blob)

7
packages/ui/src/views/chatmessage/audio-recording.js
View File

@ -2,6 +2,7 @@
* @fileoverview This file contains the API to handle audio recording.
* Originally from 'https://ralzohairi.medium.com/audio-recording-in-javascript-96eed45b75ee'
*/
import { isSafari } from 'react-device-detect'
// audio-recording.js ---------------
let microphoneButton, elapsedTimeTag
@ -277,7 +278,13 @@ export const audioRecorder = {
})
//start the recording by calling the start method on the media recorder
if (isSafari) {
// https://community.openai.com/t/whisper-problem-with-audio-mp4-blobs-from-safari/322252
// https://community.openai.com/t/whisper-api-cannot-read-files-correctly/93420/46
audioRecorder.mediaRecorder.start(1000)
} else {
audioRecorder.mediaRecorder.start()
}
})
)