diff --git a/server.js b/server.js index 67194d0..9c1946d 100644 --- a/server.js +++ b/server.js @@ -91,10 +91,23 @@ const requireAuth = (req, res, next) => { } }; -// Add user info to all templates -app.use((req, res, next) => { +// Add user info to all templates and load persistent files +app.use(async (req, res, next) => { res.locals.user = req.session.userId ? users.find(u => u.id === req.session.userId) : null; res.locals.messages = req.flash(); + + // Load user files from persistent storage if user is logged in and session doesn't have files + if (req.session.userId && (!req.session.uploadedFiles || req.session.uploadedFiles.length === 0)) { + try { + const persistentFiles = await loadUserFiles(req.session.userId); + if (persistentFiles.length > 0) { + req.session.uploadedFiles = persistentFiles; + } + } catch (error) { + console.error('Error loading user files in middleware:', error); + } + } + next(); }); @@ -206,7 +219,7 @@ function generateDocumentHash(content, originalName) { // Helper function to get document loaders from document store async function getDocumentStoreLoaders(documentStoreId) { try { - console.log(`Getting document store loaders for: ${documentStoreId}`); + const headers = { 'Content-Type': 'application/json', @@ -224,12 +237,7 @@ async function getDocumentStoreLoaders(documentStoreId) { timeout: 10000 }); - console.log('Document store details:', { - status: response.status, - id: response.data.id, - name: response.data.name, - loaders: response.data.loaders - }); + // Parse loaders - they may come as array directly or as JSON string let loaders = []; @@ -237,18 +245,18 @@ async function getDocumentStoreLoaders(documentStoreId) { if (Array.isArray(response.data.loaders)) { // Loaders are already an array loaders = response.data.loaders; - console.log('Loaders received as array:', loaders.length, 'loaders found'); + } else if (typeof response.data.loaders === 'string') { // Loaders are a JSON string that needs parsing try { loaders = JSON.parse(response.data.loaders); - console.log('Loaders parsed from JSON string:', loaders.length, 'loaders found'); + } catch (parseError) { - console.log('Could not parse loaders JSON string, will create new one'); + loaders = []; } } else { - console.log('Loaders in unexpected format, will create new one'); + loaders = []; } } @@ -271,10 +279,7 @@ async function getDocumentStoreLoaders(documentStoreId) { // Helper function to upsert document to Flowise using FormData (direct file upload) async function upsertDocumentToFlowiseFormData(fileInfo, documentMetadata) { - console.log('Starting Flowise Document Store upsert with FormData...'); - console.log(`Document: ${documentMetadata.originalName}`); - console.log(`Original file path: ${fileInfo.path}`); - console.log(`Document Store ID: ${FLOWISE_DOCUMENT_STORE_ID}`); + try { // Create FormData for file upload @@ -294,17 +299,11 @@ async function upsertDocumentToFlowiseFormData(fileInfo, documentMetadata) { // Use the first existing loader ID, but only if replaceExisting is true docId = storeInfo.loaders[0].id || storeInfo.loaders[0].loaderId; useExistingLoader = true; - console.log(`Using existing document loader ID: ${docId}`); - console.log(`Existing loader details:`, { - id: docId, - loaderName: storeInfo.loaders[0].loaderName, - splitterName: storeInfo.loaders[0].splitterName, - totalChunks: storeInfo.loaders[0].totalChunks, - status: storeInfo.loaders[0].status - }); + + } else { // Create a new loader by letting Flowise auto-generate or create new store - console.log('No existing loaders found, will create new document entry'); + } // Append form fields following the Flowise API structure @@ -346,7 +345,7 @@ async function upsertDocumentToFlowiseFormData(fileInfo, documentMetadata) { // Prepare the upsert URL const upsertUrl = `${FLOWISE_BASE_URL}/api/v1/document-store/upsert/${FLOWISE_DOCUMENT_STORE_ID}`; - console.log(`Upserting to: ${upsertUrl}`); + // Prepare headers const headers = { @@ -357,16 +356,8 @@ async function upsertDocumentToFlowiseFormData(fileInfo, documentMetadata) { headers['Authorization'] = `Bearer ${FLOWISE_API_KEY}`; } - console.log('Sending FormData upsert request...'); - console.log('FormData fields:', { - docId: docId || 'auto-generated', - fileName: fileInfo.originalName, - fileSize: fileInfo.size, - replaceExisting: useExistingLoader, - createNewDocStore: !useExistingLoader, - chunkSize: CHUNK_SIZE, - useExistingLoader: useExistingLoader - }); + + // Make the request using axios with FormData const response = await axios.post(upsertUrl, formData, { @@ -376,11 +367,7 @@ async function upsertDocumentToFlowiseFormData(fileInfo, documentMetadata) { maxBodyLength: Infinity }); - console.log('FormData upsert successful:', { - status: response.status, - statusText: response.statusText, - data: response.data - }); + return { success: true, @@ -410,7 +397,7 @@ async function upsertDocumentToFlowiseFormData(fileInfo, documentMetadata) { }); // Fall back to local storage if Flowise fails - console.log('Falling back to local storage...'); + try { const documentData = { @@ -426,7 +413,7 @@ async function upsertDocumentToFlowiseFormData(fileInfo, documentMetadata) { const documentPath = path.join(documentsDir, `${documentMetadata.documentId}.json`); await fs.writeJSON(documentPath, documentData, { spaces: 2 }); - console.log(`Document stored locally as fallback: ${documentPath}`); + return { success: true, @@ -588,13 +575,10 @@ app.get('/upload', requireAuth, (req, res) => { app.post('/upload', requireAuth, upload.single('noteFile'), async (req, res) => { try { - console.log('=== UPLOAD REQUEST START ==='); - console.log('Request file:', req.file); - console.log('Request body:', req.body); - console.log('Session userId:', req.session.userId); + if (!req.file) { - console.log('ERROR: No file uploaded'); + return res.status(400).json({ success: false, error: 'No file uploaded' @@ -613,15 +597,18 @@ app.post('/upload', requireAuth, upload.single('noteFile'), async (req, res) => status: 'processing' }; - console.log('Created fileInfo:', fileInfo); - // Store initial file info in session + + // Store initial file info in session for immediate access if (!req.session.uploadedFiles) { req.session.uploadedFiles = []; } req.session.uploadedFiles.push(fileInfo); - console.log('Total uploaded files in session:', req.session.uploadedFiles.length); + // Also store in persistent storage + await addUserFile(req.session.userId, fileInfo); + + // Send immediate response to prevent timeout res.json({ @@ -631,7 +618,7 @@ app.post('/upload', requireAuth, upload.single('noteFile'), async (req, res) => processing: true }); - console.log('Response sent, starting async processing...'); + // Process document asynchronously processDocumentAsync(fileInfo, req.session.userId, req.session); @@ -649,22 +636,15 @@ app.post('/upload', requireAuth, upload.single('noteFile'), async (req, res) => // Async function to process document and upsert to Flowise using FormData async function processDocumentAsync(fileInfo, userId, session) { try { - console.log('=== DOCUMENT PROCESSING START (FormData) ==='); - console.log(`Starting document processing for: ${fileInfo.originalName}`); - console.log(`File path: ${fileInfo.path}`); - console.log(`File size: ${fileInfo.size} bytes`); - console.log(`User ID: ${userId}`); - console.log(`Session ID in processing: ${session.id || 'undefined'}`); + // Find the file in the session to update it by reference const sessionFile = session.uploadedFiles.find(f => f.id === fileInfo.id); if (!sessionFile) { - console.error('File not found in session:', fileInfo.id); - console.error('Available files in session:', session.uploadedFiles.map(f => ({ id: f.id, name: f.originalName }))); return; } - console.log('Found session file for update:', { id: sessionFile.id, name: sessionFile.originalName, currentStatus: sessionFile.status }); + // Create document metadata const documentMetadata = { @@ -676,7 +656,7 @@ async function processDocumentAsync(fileInfo, userId, session) { fileInfo: fileInfo // Pass file info for progress tracking }; - console.log('Document metadata:', documentMetadata); + // Initialize processing result sessionFile.processingResult = { @@ -687,14 +667,13 @@ async function processDocumentAsync(fileInfo, userId, session) { status: 'uploading_to_flowise' }; - console.log('About to start FormData upsert to Flowise...'); + // Upsert document to Flowise using FormData const upsertResult = await upsertDocumentToFlowiseFormData(fileInfo, documentMetadata); - console.log('FormData upsert result:', upsertResult); - - // Update session file with processing results + + // Update session file with processing results sessionFile.status = upsertResult.success ? 'processed' : 'failed'; sessionFile.processingProgress = null; // Clear progress when done sessionFile.processingResult = { @@ -709,34 +688,31 @@ async function processDocumentAsync(fileInfo, userId, session) { if (upsertResult.errors.length > 0) { sessionFile.processingErrors = upsertResult.errors; - console.log(`Processing errors for ${sessionFile.originalName}:`, upsertResult.errors); } + + // Update persistent storage + await updateUserFile(userId, fileInfo.id, { + status: sessionFile.status, + processingResult: sessionFile.processingResult, + processingErrors: sessionFile.processingErrors + }); - console.log(`Document processing completed for: ${sessionFile.originalName}`); - console.log(`Result: FormData upload ${upsertResult.success ? 'successful' : 'failed'}`); - console.log(`Updated session file status to: ${sessionFile.status}`); + // Verify the session file was updated - console.log('Session file after update:', { id: sessionFile.id, name: sessionFile.originalName, status: sessionFile.status }); - console.log('All session files after update:', session.uploadedFiles.map(f => ({ id: f.id, name: f.originalName, status: f.status }))); + // Force session save since we modified it in an async context session.save((err) => { if (err) { console.error('Error saving session after document processing:', err); } else { - console.log('Session saved successfully after document processing'); + } }); // Log final upsert verification - console.log('Final FormData upsert verification:', { - documentStore: `${FLOWISE_BASE_URL}/api/v1/document-store/${FLOWISE_DOCUMENT_STORE_ID}`, - documentId: sessionFile.id, - fileName: sessionFile.originalName, - success: upsertResult.success, - finalStatus: sessionFile.status - }); + } catch (error) { console.error(`Error processing document ${fileInfo.originalName}:`, error); @@ -748,14 +724,21 @@ async function processDocumentAsync(fileInfo, userId, session) { sessionFile.processingError = error.message; sessionFile.processingResult = sessionFile.processingResult || {}; sessionFile.processingResult.failedAt = new Date().toISOString(); - console.log(`Updated session file status to: ${sessionFile.status} due to error`); + + // Update persistent storage + await updateUserFile(userId, fileInfo.id, { + status: 'failed', + processingError: error.message, + processingResult: sessionFile.processingResult + }); + // Force session save since we modified it in an async context session.save((err) => { if (err) { console.error('Error saving session after processing error:', err); } else { - console.log('Session saved successfully after processing error'); + } }); } @@ -806,10 +789,10 @@ app.post('/api/revise', requireAuth, async (req, res) => { if (await fs.pathExists(documentPath)) { const documentData = await fs.readJSON(documentPath); contextContent = documentData.content; - console.log(`Using stored document content for revision: ${documentData.metadata.originalName}`); + } } catch (error) { - console.log('Could not load stored document, using provided content'); + } } @@ -852,6 +835,12 @@ app.get('/chat', requireAuth, (req, res) => { req.session.chatHistory = []; } + // Initialize chat session ID if it doesn't exist + if (!req.session.chatSessionId) { + req.session.chatSessionId = `educat-${req.session.userId}-${Date.now()}`; + + } + res.render('chat', { title: 'Chat with EduCat AI', chatHistory: req.session.chatHistory @@ -867,17 +856,41 @@ app.post('/api/chat', requireAuth, async (req, res) => { req.session.chatHistory = []; } - console.log('Chat message received:', message); - console.log('Current chat history length:', req.session.chatHistory.length); + // Initialize or get persistent chat session ID for this user + if (!req.session.chatSessionId) { + req.session.chatSessionId = `${req.session.userId}-${Date.now()}`; + + } - // Call Flowise API for chat with session history - const response = await axios.post(`${FLOWISE_API_URL}/${FLOWISE_CHATFLOW_ID}`, { + + + // Prepare the request payload for Flowise with sessionId and chatId + const flowisePayload = { question: message, - history: req.session.chatHistory - }); + sessionId: req.session.chatSessionId + }; + + // Add chatId if we have one from previous conversations + if (req.session.chatId) { + flowisePayload.chatId = req.session.chatId; + + } + + + + // Call Flowise API for chat with session history and sessionId + const response = await axios.post(`${FLOWISE_API_URL}/${FLOWISE_CHATFLOW_ID}`, flowisePayload); + + const aiResponse = response.data.text || response.data.answer || 'No response received'; + // Save the chatId from Flowise response for future requests + if (response.data.chatId) { + req.session.chatId = response.data.chatId; + + } + // Add the conversation to session history req.session.chatHistory.push({ human: message, @@ -891,7 +904,7 @@ app.post('/api/chat', requireAuth, async (req, res) => { } }); - console.log('Updated chat history length:', req.session.chatHistory.length); + res.json({ success: true, @@ -911,7 +924,7 @@ app.post('/api/chat', requireAuth, async (req, res) => { app.get('/api/chat/history', requireAuth, (req, res) => { try { const chatHistory = req.session.chatHistory || []; - console.log('Chat history requested, returning', chatHistory.length, 'messages'); + res.json({ success: true, @@ -931,6 +944,11 @@ app.get('/api/chat/history', requireAuth, (req, res) => { app.delete('/api/chat/history', requireAuth, (req, res) => { try { req.session.chatHistory = []; + // Reset the session ID to start a fresh conversation + req.session.chatSessionId = `${req.session.userId}-${Date.now()}`; + // Clear the Flowise chatId + delete req.session.chatId; + req.session.save((err) => { if (err) { console.error('Error clearing chat session:', err); @@ -940,7 +958,7 @@ app.delete('/api/chat/history', requireAuth, (req, res) => { }); } - console.log('Chat history cleared for user:', req.session.userId); + res.json({ success: true, message: 'Chat history cleared' @@ -956,12 +974,36 @@ app.delete('/api/chat/history', requireAuth, (req, res) => { } }); -app.get('/dashboard', requireAuth, (req, res) => { - const files = req.session.uploadedFiles || []; - res.render('dashboard', { - title: 'Dashboard - EduCat', - files: files - }); +app.get('/dashboard', requireAuth, async (req, res) => { + try { + // Load persistent files for this user + const persistentFiles = await loadUserFiles(req.session.userId); + + // Merge with session files (in case there are newly uploaded files not yet saved) + const sessionFiles = req.session.uploadedFiles || []; + const allFiles = [...persistentFiles]; + + // Add any session files that aren't already in persistent storage + sessionFiles.forEach(sessionFile => { + if (!persistentFiles.find(f => f.id === sessionFile.id)) { + allFiles.push(sessionFile); + } + }); + + // Update session with merged files for current session use + req.session.uploadedFiles = allFiles; + + res.render('dashboard', { + title: 'Dashboard - EduCat', + files: allFiles + }); + } catch (error) { + console.error('Error loading dashboard:', error); + res.render('dashboard', { + title: 'Dashboard - EduCat', + files: req.session.uploadedFiles || [] + }); + } }); // File management endpoints @@ -1020,7 +1062,10 @@ app.delete('/api/files/:fileId', requireAuth, async (req, res) => { // Remove from session req.session.uploadedFiles.splice(fileIndex, 1); - console.log(`File deleted: ${file.originalName} (ID: ${fileId})`); + // Remove from persistent storage + await removeUserFile(req.session.userId, fileId); + + res.json({ success: true, @@ -1040,14 +1085,22 @@ app.delete('/api/files/:fileId', requireAuth, async (req, res) => { app.get('/api/files/:fileId/status', requireAuth, async (req, res) => { try { const fileId = req.params.fileId; - const files = req.session.uploadedFiles || []; - const file = files.find(f => f.id === fileId); + let files = req.session.uploadedFiles || []; + let file = files.find(f => f.id === fileId); - console.log(`Status requested for file ${fileId}`); - console.log(`Session ID: ${req.session.id || req.sessionID}`); - console.log(`User ID: ${req.session.userId}`); - console.log(`Files in session:`, files.map(f => ({ id: f.id, name: f.originalName, status: f.status }))); - console.log(`Found file:`, file ? { id: file.id, name: file.originalName, status: file.status } : 'Not found'); + // If not in session, try to load from persistent storage + if (!file) { + const persistentFiles = await loadUserFiles(req.session.userId); + file = persistentFiles.find(f => f.id === fileId); + + // If found in persistent storage, add to session for future requests + if (file) { + if (!req.session.uploadedFiles) { + req.session.uploadedFiles = []; + } + req.session.uploadedFiles.push(file); + } + } if (!file) { return res.status(404).json({ success: false, error: 'File not found' }); @@ -1116,10 +1169,24 @@ app.post('/api/files/:fileId/retry', requireAuth, async (req, res) => { // Bulk processing status endpoint app.get('/api/files/status/all', requireAuth, async (req, res) => { try { - const files = req.session.uploadedFiles || []; - console.log('Status check requested for session files:', files.map(f => ({ id: f.id, name: f.originalName, status: f.status }))); + // Load persistent files and merge with session + const persistentFiles = await loadUserFiles(req.session.userId); + const sessionFiles = req.session.uploadedFiles || []; - const statusSummary = files.map(file => ({ + // Merge files, preferring session data for current uploads + const allFiles = [...persistentFiles]; + sessionFiles.forEach(sessionFile => { + const existingIndex = allFiles.findIndex(f => f.id === sessionFile.id); + if (existingIndex !== -1) { + // Update existing with session data (more current) + allFiles[existingIndex] = sessionFile; + } else { + // Add new session file + allFiles.push(sessionFile); + } + }); + + const statusSummary = allFiles.map(file => ({ id: file.id, originalName: file.originalName, status: file.status, @@ -1133,15 +1200,13 @@ app.get('/api/files/status/all', requireAuth, async (req, res) => { })); const summary = { - totalFiles: files.length, - processing: files.filter(f => f.status === 'processing').length, - processed: files.filter(f => f.status === 'processed').length, - failed: files.filter(f => f.status === 'failed').length, + totalFiles: allFiles.length, + processing: allFiles.filter(f => f.status === 'processing').length, + processed: allFiles.filter(f => f.status === 'processed').length, + failed: allFiles.filter(f => f.status === 'failed').length, files: statusSummary }; - console.log('Returning status summary:', summary); - res.json({ success: true, summary: summary @@ -1163,15 +1228,12 @@ app.get('/quiz', requireAuth, (req, res) => { }); app.post('/api/generate-quiz', requireAuth, async (req, res) => { - console.log('=== QUIZ API REQUEST RECEIVED ==='); - console.log('Request body:', req.body); - console.log('User ID:', req.session.userId); - console.log('================================='); + try { const { topic, difficulty, questionCount, quizType } = req.body; - console.log('Quiz request:', { topic, difficulty, questionCount, quizType }); + let prompt = ''; switch (quizType) { @@ -1217,7 +1279,7 @@ app.post('/api/generate-quiz', requireAuth, async (req, res) => { prompt = `Generate exactly ${questionCount} multiple choice questions about "${topic}" at ${difficulty} difficulty level.`; } - console.log('Sending prompt to Flowise:', prompt.substring(0, 100) + '...'); + // Call Flowise API const response = await axios.post(`${FLOWISE_API_URL}/${FLOWISE_CHATFLOW_ID}`, { @@ -1225,16 +1287,12 @@ app.post('/api/generate-quiz', requireAuth, async (req, res) => { history: [] }); - console.log('Flowise response received:', { - status: response.status, - dataType: typeof response.data, - dataPreview: JSON.stringify(response.data).substring(0, 200) + '...' - }); + let quizData; try { const responseText = response.data.text || response.data.answer || response.data; - console.log('Raw response text preview:', responseText.substring(0, 500) + '...'); + // Try to extract JSON from the response let jsonString = null; @@ -1243,7 +1301,7 @@ app.post('/api/generate-quiz', requireAuth, async (req, res) => { const codeBlockMatch = responseText.match(/```(?:json)?\s*(\[[\s\S]*?\])\s*```/); if (codeBlockMatch) { jsonString = codeBlockMatch[1]; - console.log('Found JSON in code block'); + } else { // Try to find JSON array by counting brackets const startIndex = responseText.indexOf('['); @@ -1262,29 +1320,27 @@ app.post('/api/generate-quiz', requireAuth, async (req, res) => { if (bracketCount === 0) { jsonString = responseText.substring(startIndex, endIndex + 1); - console.log('Found JSON by bracket counting'); + } } } if (jsonString) { - console.log('Parsing JSON string:', jsonString.substring(0, 200) + '...'); quizData = JSON.parse(jsonString); - console.log('Successfully parsed quiz data, questions:', quizData.length); } else { - console.log('No JSON found, using fallback quiz'); quizData = generateFallbackQuiz(topic, questionCount, quizType); } } catch (parseError) { console.error('Quiz parsing error:', parseError); - console.log('Using fallback quiz due to parsing error'); quizData = generateFallbackQuiz(topic, questionCount, quizType); } - console.log('Final quiz data:', { - questionsCount: quizData.length, - firstQuestion: quizData[0] - }); + // Ensure quizData is always defined and is an array + if (!quizData || !Array.isArray(quizData) || quizData.length === 0) { + quizData = generateFallbackQuiz(topic, questionCount, quizType); + } + + res.json({ success: true, @@ -1300,8 +1356,6 @@ app.post('/api/generate-quiz', requireAuth, async (req, res) => { // Return fallback quiz on error const fallbackQuiz = generateFallbackQuiz(req.body.topic || 'General Knowledge', req.body.questionCount || 5, req.body.quizType || 'multiple-choice'); - - console.log('Returning fallback quiz due to error'); res.json({ success: true, quiz: fallbackQuiz, @@ -1496,7 +1550,7 @@ app.post('/api/submit-quiz', requireAuth, async (req, res) => { // Store quiz result persistently await addQuizResult(req.session.userId, quizResult); - console.log(`Quiz result saved for user ${req.session.userId}: ${score}/${quiz.length} (${percentage}%)`); + res.json({ success: true, @@ -1515,6 +1569,65 @@ app.post('/api/submit-quiz', requireAuth, async (req, res) => { } }); +// User file storage persistence +const USER_FILES_DIR = path.join(__dirname, 'data', 'user-files'); + +// Ensure user files directory exists +async function ensureUserFilesDirectory() { + await fs.ensureDir(USER_FILES_DIR); +} + +// Save user files to persistent storage +async function saveUserFiles(userId, files) { + try { + await ensureUserFilesDirectory(); + const userFilePath = path.join(USER_FILES_DIR, `user-${userId}-files.json`); + await fs.writeJSON(userFilePath, files, { spaces: 2 }); + } catch (error) { + console.error('Error saving user files:', error); + } +} + +// Load user files from persistent storage +async function loadUserFiles(userId) { + try { + await ensureUserFilesDirectory(); + const userFilePath = path.join(USER_FILES_DIR, `user-${userId}-files.json`); + if (await fs.pathExists(userFilePath)) { + const files = await fs.readJSON(userFilePath); + return files || []; + } + return []; + } catch (error) { + console.error('Error loading user files:', error); + return []; + } +} + +// Add a file to user's persistent storage +async function addUserFile(userId, fileInfo) { + const userFiles = await loadUserFiles(userId); + userFiles.push(fileInfo); + await saveUserFiles(userId, userFiles); +} + +// Update a file in user's persistent storage +async function updateUserFile(userId, fileId, updates) { + const userFiles = await loadUserFiles(userId); + const fileIndex = userFiles.findIndex(f => f.id === fileId); + if (fileIndex !== -1) { + userFiles[fileIndex] = { ...userFiles[fileIndex], ...updates }; + await saveUserFiles(userId, userFiles); + } +} + +// Remove a file from user's persistent storage +async function removeUserFile(userId, fileId) { + const userFiles = await loadUserFiles(userId); + const filteredFiles = userFiles.filter(f => f.id !== fileId); + await saveUserFiles(userId, filteredFiles); +} + // Quiz results persistence const QUIZ_RESULTS_FILE = path.join(__dirname, 'data', 'quiz-results.json'); @@ -1603,7 +1716,7 @@ app.delete('/api/quiz-history', requireAuth, async (req, res) => { allResults[userId] = []; await saveQuizResults(allResults); - console.log(`Quiz history cleared for user ${userId}`); + res.json({ success: true, @@ -1750,9 +1863,6 @@ app.use((req, res) => { app.listen(PORT, () => { console.log(`EduCat server running on http://localhost:${PORT}`); - console.log(`Flowise API URL: ${FLOWISE_API_URL}`); - console.log(`Flowise Chatflow ID: ${FLOWISE_CHATFLOW_ID}`); - console.log(`Flowise Document Store ID: ${FLOWISE_DOCUMENT_STORE_ID}`); });