Merge pull request #166 from CodeForPhilly/feat/zbl-enahancedsearch

Feat/zbl enahancedsearch

Author: ZacharyLeahan

Dockerfile

@@ -1,7 +1,10 @@
-FROM node@sha256:426f843809ae05f324883afceebaa2b9cab9cb697097dbb1a2a7a41c5701de72
+FROM node:20-slim
 
 # Install ImageMagick and AWS CLI
-RUN apk add --no-cache imagemagick aws-cli
+RUN apt-get update && apt-get install -y \
+    imagemagick \
+    awscli \
+    && rm -rf /var/lib/apt/lists/*
 
 # Set NODE_OPTIONS for OpenSSL compatibility and limit memory usage
 ENV NODE_OPTIONS="--openssl-legacy-provider --max-old-space-size=768"

app.js

@@ -1,9 +1,45 @@
 const db = require('./lib/db');
 const server = require('./lib/server');
+const { generateEmbeddings } = require('./scripts/generate-embeddings');
 
 run();
 
 async function run() {
-  const { plants, nurseries } = await db();
-  await server({ plants, nurseries });
+  const { plants, nurseries, close } = await db();
+  
+  // Check if embeddings exist
+  const embeddingCount = await plants.countDocuments({ 
+    embedding: { $exists: true, $ne: null } 
+  });
+  
+  if (embeddingCount === 0) {
+    console.log('\n⚠️  No embeddings found in database.');
+    console.log('🔄 Auto-generating embeddings... This may take a few minutes.\n');
+    
+    try {
+      // Close the current connection since generateEmbeddings will create its own
+      await close();
+      
+      // Generate embeddings (this will create its own DB connection)
+      await generateEmbeddings();
+      
+      console.log('\n✅ Embeddings generated successfully!');
+      console.log('🔄 Reconnecting to database...\n');
+      
+      // Reconnect to database
+      const dbConnection = await db();
+      await server({ plants: dbConnection.plants, nurseries: dbConnection.nurseries });
+    } catch (error) {
+      console.error('\n❌ Error generating embeddings:', error);
+      console.error('⚠️  Server will start without embeddings. Semantic search will not work.');
+      console.error('💡 Run manually: npm run generate-embeddings\n');
+      
+      // Start server anyway (without embeddings)
+      const dbConnection = await db();
+      await server({ plants: dbConnection.plants, nurseries: dbConnection.nurseries });
+    }
+  } else {
+    console.log(`✅ Found ${embeddingCount} plants with embeddings.`);
+    await server({ plants, nurseries });
+  }
 }

lib/embeddings.js

@@ -2,11 +2,53 @@
 let transformersModule = null;
 let embeddingModel = null;
 
+/**
+ * Ensure global.fetch is properly overridden to fix malformed URLs
+ * This is a backup in case the SSR bundle overwrites it
+ */
+function ensureFetchOverride() {
+  if (typeof global !== 'undefined') {
+    const nodeFetch = require('node-fetch');
+    
+    const fixedFetch = function(url, options) {
+      let fetchUrl = url;
+      
+      if (typeof url === 'string') {
+        // Fix malformed URLs like "http://localhost:3000https://huggingface.co/..."
+        if (url.includes('localhost') && url.includes('huggingface.co')) {
+          // Extract the actual HuggingFace URL (everything from https:// onwards)
+          const httpsMatch = url.match(/https?:\/\/[^/]*huggingface\.co\/.*/);
+          if (httpsMatch) {
+            fetchUrl = httpsMatch[0];
+          } else {
+            // Fallback: find https:// in the string and use everything from there
+            const httpsIndex = url.indexOf('https://');
+            if (httpsIndex !== -1) {
+              fetchUrl = url.substring(httpsIndex);
+            }
+          }
+        }
+      }
+      
+      // Use node-fetch for all requests (more reliable in Node.js environment)
+      return nodeFetch(fetchUrl, options);
+    };
+    
+    // Always override to ensure it's set correctly
+    global.fetch = fixedFetch;
+    if (typeof globalThis !== 'undefined') {
+      globalThis.fetch = fixedFetch;
+    }
+  }
+}
+
 /**
  * Get the transformers module using dynamic import (required for ES modules)
  */
 async function getTransformersModule() {
   if (!transformersModule) {
+    // Ensure fetch is overridden before importing transformers
+    ensureFetchOverride();
     transformersModule = await import('@xenova/transformers');
   }
   return transformersModule;
@@ -20,6 +62,10 @@ async function getEmbeddingModel() {
   if (!embeddingModel) {
     console.log('Loading embedding model...');
     const { pipeline } = await getTransformersModule();
+    
+    // global.fetch is already overridden at the top of this file
+    // to fix malformed URLs from SSR webpack polyfills
+    
     embeddingModel = await pipeline(
       'feature-extraction',
       'Xenova/all-MiniLM-L6-v2' // Lightweight model, ~80MB, good for semantic search

lib/server.js

@@ -16,6 +16,39 @@ if (!isDevelopment) {
     manifest = require("../ssr/ssr-manifest.json");
     const appPath = path.join(__dirname, "../ssr", manifest["app.js"]);
     renderer = require(appPath).default;
+    
+    // Override global.fetch AFTER SSR bundle loads to fix malformed URLs
+    // The SSR bundle sets its own fetch that prepends localhost:3000 to HuggingFace URLs
+    const nodeFetch = require('node-fetch');
+    const fixedFetch = function(url, options) {
+      let fetchUrl = url;
+      
+      if (typeof url === 'string') {
+        // Fix malformed URLs like "http://localhost:3000https://huggingface.co/..."
+        if (url.includes('localhost') && url.includes('huggingface.co')) {
+          // Extract the actual HuggingFace URL (everything from https:// onwards)
+          const httpsMatch = url.match(/https?:\/\/[^/]*huggingface\.co\/.*/);
+          if (httpsMatch) {
+            fetchUrl = httpsMatch[0];
+          } else {
+            // Fallback: find https:// in the string and use everything from there
+            const httpsIndex = url.indexOf('https://');
+            if (httpsIndex !== -1) {
+              fetchUrl = url.substring(httpsIndex);
+            }
+          }
+        }
+      }
+      
+      // Use node-fetch for all requests (more reliable in Node.js environment)
+      return nodeFetch(fetchUrl, options);
+    };
+    
+    // Override on both global and globalThis to ensure it's caught
+    global.fetch = fixedFetch;
+    if (typeof globalThis !== 'undefined') {
+      globalThis.fetch = fixedFetch;
+    }
   } catch (e) {
     console.warn("SSR build not found, running in development mode");
   }
@@ -557,6 +590,7 @@ module.exports = async function ({ plants, nurseries }) {
         // Fetch all candidates (we'll calculate similarity and sort in JS)
         // For performance, we might want to limit this, but for now get all matching filters
         const candidates = await plants.find(baseQuery).toArray();
+        console.log(`Semantic search: Found ${candidates.length} candidates with embeddings`);
 
         // Calculate similarity scores
         const queryEmbedding = query._queryEmbedding;
@@ -573,7 +607,12 @@ module.exports = async function ({ plants, nurseries }) {
               _semanticScore: similarity
             };
           })
-          .filter(plant => plant && plant._semanticScore >= 0.3) // Minimum similarity threshold
+          .filter(plant => {
+            if (!plant || plant._semanticScore < 0.3) {
+              return false;
+            }
+            return true;
+          }) // Minimum similarity threshold
           .sort((a, b) => {
             // If user wants to sort by Search Relevance, prioritize semantic score
             if (originalSortKeys.includes("_semanticScore") || req.query.sort === "Sort by Search Relevance") {

scripts/generate-embeddings.js

@@ -72,16 +72,30 @@ async function generateEmbeddings() {
   }
 
   await close();
-  process.exit(0);
+  return { processed, updated, skipped };
 }
 
-// Handle errors
-process.on('unhandledRejection', (error) => {
-  console.error('Unhandled error:', error);
-  process.exit(1);
-});
+// Export for use in other modules
+module.exports = { generateEmbeddings };
+
+// If run directly as a script, execute it
+if (require.main === module) {
+  // Handle errors for standalone execution
+  process.on('unhandledRejection', (error) => {
+    console.error('Unhandled error:', error);
+    process.exit(1);
+  });
+
+  generateEmbeddings()
+    .then(() => {
+      process.exit(0);
+    })
+    .catch((error) => {
+      console.error('Unhandled error:', error);
+      process.exit(1);
+    });
+}
 
-generateEmbeddings();