Translation algorithm improvements

README.md

@@ -5,7 +5,7 @@ colorFrom: purple
 colorTo: yellow
 sdk: docker
 pinned: false
-short_description: Document authoring and translation to English
+short_description: A WYSIWYG editor that translates text to multiple languages.
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

document-authoring.js

@@ -2,9 +2,9 @@
 const app = document.getElementById('app');
 app.innerHTML = `
   <div style="height: 100vh; display: flex; flex-direction: column;">
-    <div style="padding: 20px;">
+    <div style="background: #f5f5f5; border-bottom: 1px solid #ddd; padding: 10px;">
       <div id="statusIndicator">
-        <span class="spinner"></span> Services starting...
+        <span class="spinner"></span> Translation services starting...
       </div>
       <div id="translationControls">
         <select id="languageSelect">
@@ -15,10 +15,59 @@ app.innerHTML = `
         <button id="translateButton">Translate Document</button>
       </div>
     </div>
-    <div id="editor" style="flex: 1; border: 1px solid black; min-height: 0;"></div>
+    <div style="flex: 1; display: flex; flex-direction: column; position: relative;">
+      <div id="editor" style="position: absolute; top: 0; left: 0; right: 0; bottom: 0;"></div>
+    </div>
   </div>
 `;
 
+// Add retry count
+let retryCount = 0;
+const MAX_RETRIES = 10; // Will try for 20 seconds (10 attempts * 2 second interval)
+
+function checkServicesStatus() {
+  fetch('/healthcheck')
+    .then(response => {
+      if (response.ok) {
+        window.servicesReady = true;
+        const translationControls = document.getElementById('translationControls');
+        const statusIndicator = document.getElementById('statusIndicator');
+        if (translationControls) {
+          translationControls.style.display = 'block';
+        }
+        if (statusIndicator) {
+          statusIndicator.style.display = 'none';
+        }
+        return true;
+      }
+      throw new Error('Services not ready');
+    })
+    .catch(error => {
+      retryCount++;
+      const statusIndicator = document.getElementById('statusIndicator');
+      if (statusIndicator) {
+        if (retryCount >= MAX_RETRIES) {
+          statusIndicator.innerHTML = '❌ Failed to initialize translation services. Try restarting the space.';
+          statusIndicator.style.color = '#dc3545';
+          clearInterval(statusInterval);
+        } else {
+          statusIndicator.innerHTML = `<span class="spinner"></span> Translation services starting... (attempt ${retryCount}/${MAX_RETRIES})`;
+        }
+      }
+      console.log('Waiting for services...', error);
+      return false;
+    });
+}
+
+// Check status every 2 seconds until ready or max retries reached
+const statusInterval = setInterval(() => {
+  if (window.servicesReady || retryCount >= MAX_RETRIES) {
+    clearInterval(statusInterval);
+  } else {
+    checkServicesStatus();
+  }
+}, 2000);
+
 // Load Document Authoring SDK
 const script = document.createElement('script');
 script.src = 'https://document-authoring-cdn.pspdfkit.com/releases/document-authoring-1.0.26-umd.js';
@@ -35,7 +84,42 @@ script.onload = async () => {
     }
   );
 
-  // Translation function with language parameter
+  // Helper function to extract text runs with their paths
+  function flattenTextRuns(obj, path = [], result = []) {
+    for (let key in obj) {
+      const newPath = [...path, key];
+      if (typeof obj[key] === "string" && key === "text" && obj[key].length > 0) {
+        result.push({ text: obj[key], path: newPath });
+      } else if (typeof obj[key] === "object") {
+        flattenTextRuns(obj[key], newPath, result);
+      }
+    }
+    return result;
+  }
+
+  // Helper function to group adjacent text runs
+  function groupAdjacentRuns(textRuns, maxChunkSize = 1000) {
+    const groups = [];
+    let currentGroup = [];
+    let currentSize = 0;
+
+    for (let run of textRuns) {
+      if (currentSize + run.text.length > maxChunkSize && currentGroup.length > 0) {
+        groups.push([...currentGroup]);
+        currentGroup = [];
+        currentSize = 0;
+      }
+      currentGroup.push(run);
+      currentSize += run.text.length;
+    }
+    
+    if (currentGroup.length > 0) {
+      groups.push(currentGroup);
+    }
+    
+    return groups;
+  }
+
   async function translate(content, lang) {
     try {
       const response = await fetch('/client/api/v1/chat/completions', {
@@ -47,11 +131,11 @@ script.onload = async () => {
           messages: [
             {
               role: "system",
-              content: "You are a translator. Only provide the translation, no explanations or additional text."
+              content: `Translate the following text to ${lang}. Each segment is marked with ||| delimiters. Translate each segment independently while preserving original capitalization and punctuation. Add spaces at the start or end of segments only if they exist in the original text.`
             },
             {
               role: "user",
-              content: `Translate the following text to ${lang}: ${content}`
+              content: content
             }
           ],
           model: "gemma-2b",
@@ -71,14 +155,50 @@ script.onload = async () => {
     }
   }
 
-  // Function to recursively translate text in the JSON structure
-  async function translateRecursive(obj, lang) {
-    for (let key in obj) {
-      if (typeof obj[key] === "string" && key === "text" && obj[key].length > 0) {
-        obj[key] = await translate(obj[key], lang);
-      } else if (typeof obj[key] === "object") {
-        await translateRecursive(obj[key], lang);
-      }
+  // Function to update nested object value by path
+  function setNestedValue(obj, path, value) {
+    let current = obj;
+    for (let i = 0; i < path.length - 1; i++) {
+      current = current[path[i]];
+    }
+    current[path[path.length - 1]] = value;
+  }
+
+  // Enhanced recursive translation function
+  async function translateRecursive(docJson, lang) {
+    const textRuns = flattenTextRuns(docJson);
+    const groups = groupAdjacentRuns(textRuns);
+    
+    for (let group of groups) {
+      // Prepare content for translation, preserving spaces
+      const content = group.map(run => `|||${run.text}|||`).join('\n');
+      
+      // Get translations
+      const translatedContent = await translate(content, lang);
+      
+      // Parse translations
+      const translations = translatedContent
+        .split('|||')
+        .filter(t => t.trim())
+        .map(t => t.trim());
+      
+      // Update document
+      group.forEach((run, index) => {
+        if (translations[index]) {
+          // Preserve original leading/trailing spaces
+          const originalText = run.text;
+          let translatedText = translations[index];
+          
+          if (originalText.startsWith(' ')) {
+            translatedText = ' ' + translatedText;
+          }
+          if (originalText.endsWith(' ')) {
+            translatedText = translatedText + ' ';
+          }
+          
+          setNestedValue(docJson, run.path, translatedText);
+        }
+      });
     }
   }
 

index.html

@@ -37,7 +37,6 @@
 
       #translationControls {
         display: none;
-        margin-bottom: 10px;
       }
 
       #languageSelect {
@@ -61,47 +60,6 @@
   </head>  
   <body>
     <div id="app"></div>
-    <script>
-      // Add this before loading the main script
-      window.servicesReady = false;
-      
-      // Create a function to check service status
-      function checkServicesStatus() {
-        fetch('/healthcheck')
-          .then(response => {
-            if (response.ok) {
-              window.servicesReady = true;
-              const translationControls = document.getElementById('translationControls');
-              const statusIndicator = document.getElementById('statusIndicator');
-              if (translationControls) {
-                translationControls.style.display = 'block';
-              }
-              if (statusIndicator) {
-                statusIndicator.style.display = 'none';
-              }
-              return true;
-            }
-            throw new Error('Services not ready');
-          })
-          .catch(error => {
-            const statusIndicator = document.getElementById('statusIndicator');
-            if (statusIndicator) {
-              statusIndicator.innerHTML = '<span class="spinner"></span> Services starting...';
-            }
-            console.log('Waiting for services...', error);
-            return false;
-          });
-      }
-
-      // Check status every 2 seconds until ready
-      const statusInterval = setInterval(() => {
-        if (window.servicesReady) {
-          clearInterval(statusInterval);
-        } else {
-          checkServicesStatus();
-        }
-      }, 2000);
-    </script>
     <script src="/document-authoring.js"></script>
   </body>
 </html>