Merge pull request #255 from rishi-raj-jain/main

handle failing
Cap-go · Oct 16, 2024 · 67f838e · 67f838e
2 parents 3232740 + 8a73396
commit 67f838e
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 10 deletions.
diff --git a/scripts/generate_blog_translations.tsx b/scripts/generate_blog_translations.tsx
@@ -15,6 +15,7 @@ for (const lang of languages) {
   const langBlogDirectory = join(contentDirectory, lang, 'blog')
   if (!existsSync(langBlogDirectory)) mkdirSync(langBlogDirectory, { recursive: true })
   const blogFiles = readdirSync(blogDirectory)
+  const failedTranslations: { [file: string]: boolean } = {}
   const processFile = async (file: string) => {
     const filePath = join(blogDirectory, file)
     const destinationPath = join(langBlogDirectory, file)
@@ -26,14 +27,17 @@ for (const lang of languages) {
     if (grayMatterJson.title) {
       const translatedTitle = await translateText(grayMatterJson.title, lang)
       if (translatedTitle) grayMatterJson['title'] = translatedTitle
+      else failedTranslations[file] = true
     }
     if (grayMatterJson.description) {
       const translatedDescription = await translateText(grayMatterJson.description, lang)
       if (translatedDescription) grayMatterJson['description'] = translatedDescription
+      else failedTranslations[file] = true
     }
     if (grayMatterJson.head_image_alt) {
       const translatedHeadImageAlt = await translateText(grayMatterJson.head_image_alt, lang)
       if (translatedHeadImageAlt) grayMatterJson['head_image_alt'] = translatedHeadImageAlt
+      else failedTranslations[file] = true
     }
     grayMatterJson['locale'] = lang
     appendFileSync(destinationPath, matter.stringify('', grayMatterJson), 'utf8')
@@ -50,12 +54,14 @@ for (const lang of languages) {
       if ((currentChunk + sentence).length > 4000) {
         const tmp = await translateText(currentChunk, lang)
         if (tmp) appendFileSync(destinationPath, tmp, 'utf8')
+        else failedTranslations[file] = true
         currentChunk = sentence
       } else currentChunk += sentence
     }
     if (currentChunk) {
       const tmp = await translateText(currentChunk, lang)
       if (tmp) appendFileSync(destinationPath, tmp, 'utf8')
+      else failedTranslations[file] = true
     }
     let translatedContent = readFileSync(destinationPath, 'utf8')
     codeBlocks.forEach((match, _) => {
@@ -74,4 +80,11 @@ for (const lang of languages) {
     await Promise.all(promises)
     await new Promise((resolve) => setTimeout(resolve, 1000))
   }
+  const failedFiles = Object.keys(failedTranslations)
+  if (failedFiles.length > 0) {
+    console.log(`Retrying failed translations for ${lang}...`)
+    for (const file of failedFiles) {
+      await processFile(file)
+    }
+  }
 }
diff --git a/scripts/generate_plugin_translations.tsx b/scripts/generate_plugin_translations.tsx
@@ -15,6 +15,7 @@ for (const lang of languages) {
   const langBlogDirectory = join(contentDirectory, lang, 'plugins-tutorials')
   if (!existsSync(langBlogDirectory)) mkdirSync(langBlogDirectory, { recursive: true })
   const blogFiles = readdirSync(blogDirectory)
+  const failedTranslations: { [file: string]: boolean } = {}
   const processFile = async (file: string) => {
     const filePath = join(blogDirectory, file)
     const destinationPath = join(langBlogDirectory, file)
@@ -26,14 +27,17 @@ for (const lang of languages) {
     if (grayMatterJson.title) {
       const translatedTitle = await translateText(grayMatterJson.title, lang)
       if (translatedTitle) grayMatterJson['title'] = translatedTitle
+      else failedTranslations[file] = true
     }
     if (grayMatterJson.description) {
-      const translatedTitle = await translateText(grayMatterJson.description, lang)
-      if (translatedTitle) grayMatterJson['description'] = translatedTitle
+      const translatedDescription = await translateText(grayMatterJson.description, lang)
+      if (translatedDescription) grayMatterJson['description'] = translatedDescription
+      else failedTranslations[file] = true
     }
     if (grayMatterJson.head_image_alt) {
       const translatedHeadImageAlt = await translateText(grayMatterJson.head_image_alt, lang)
       if (translatedHeadImageAlt) grayMatterJson['head_image_alt'] = translatedHeadImageAlt
+      else failedTranslations[file] = true
     }
     grayMatterJson['locale'] = lang
     appendFileSync(destinationPath, matter.stringify('', grayMatterJson), 'utf8')
@@ -50,12 +54,14 @@ for (const lang of languages) {
       if ((currentChunk + sentence).length > 4000) {
         const tmp = await translateText(currentChunk, lang)
         if (tmp) appendFileSync(destinationPath, tmp, 'utf8')
+        else failedTranslations[file] = true
         currentChunk = sentence
       } else currentChunk += sentence
     }
     if (currentChunk) {
       const tmp = await translateText(currentChunk, lang)
       if (tmp) appendFileSync(destinationPath, tmp, 'utf8')
+      else failedTranslations[file] = true
     }
     let translatedContent = readFileSync(destinationPath, 'utf8')
     codeBlocks.forEach((match, _) => {
@@ -74,4 +80,11 @@ for (const lang of languages) {
     await Promise.all(promises)
     await new Promise((resolve) => setTimeout(resolve, 1000))
   }
+  const failedFiles = Object.keys(failedTranslations)
+  if (failedFiles.length > 0) {
+    console.log(`Retrying failed translations for ${lang}...`)
+    for (const file of failedFiles) {
+      await processFile(file)
+    }
+  }
 }
diff --git a/scripts/translate.tsx b/scripts/translate.tsx
@@ -1,14 +1,46 @@
 import 'dotenv/config'
 
+// export const translateText = async (text: string, lang: string) => {
+//   const response = await fetch('https://api.openai.com/v1/chat/completions', {
+//     method: 'POST',
+//     headers: {
+//       'Content-Type': 'application/json',
+//       Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+//     },
+//     body: JSON.stringify({
+//       model: 'gpt-4o-mini',
+//       messages: [
+//         {
+//           role: 'system',
+//           content: `Only respond with the translation of the text. No other or unrelated text or characters. Make sure to avoid translating links, HTML tags, code blocks, image links.`,
+//         },
+//         {
+//           role: 'user',
+//           content: `Translate the following text to ${lang} locale:\n\n${text}`,
+//         },
+//       ],
+//       max_tokens: 4000,
+//     }),
+//   })
+//   if (response.status !== 200) {
+//     console.error(response.statusText)
+//     return null
+//   }
+//   const data = await response.json()
+//   return data.choices[0].message.content.trim()
+// }
+
 export const translateText = async (text: string, lang: string) => {
-  const response = await fetch('https://api.openai.com/v1/chat/completions', {
+  const response = await fetch('https://api.anthropic.com/v1/messages', {
     method: 'POST',
     headers: {
+      'anthropic-version': '2023-06-01',
       'Content-Type': 'application/json',
-      Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+      'x-api-key': `${process.env.ANTHROPIC_API_KEY}`,
     },
     body: JSON.stringify({
-      model: 'gpt-4o-mini',
+      max_tokens: 4000,
+      model: 'claude-3-5-sonnet-20240620',
       messages: [
         {
           role: 'system',
@@ -19,15 +51,14 @@ export const translateText = async (text: string, lang: string) => {
           content: `Translate the following text to ${lang} locale:\n\n${text}`,
         },
       ],
-      max_tokens: 4000,
     }),
   })
   if (response.status !== 200) {
     console.error(response.statusText)
     return null
   }
   const data = await response.json()
-  return data.choices[0].message.content.trim()
+  return data.content[0].text
 }
 
 // export const translateText = async (text: string, lang: string) => {

diff --git a/src/components/Footer.vue b/src/components/Footer.vue
@@ -249,9 +249,14 @@ const navigation = {
               aria-labelledby="options-menu"
             >
               <div class="py-1" role="none">
-                <a v-for="item in locales" :href="getRelativeLocaleUrl(item.toLowerCase(), currentPath)" class="block px-4 py-2 text-sm text-gray-700 hover:bg-gray-100 hover:text-gray-900" role="menuitem">{{
-                  item.toUpperCase()
-                }}</a>
+                <a
+                  v-for="item in locales"
+                  :href="getRelativeLocaleUrl(item.toLowerCase(), currentPath)"
+                  role="menuitem"
+                  class="block px-4 py-2 text-sm text-gray-700 hover:bg-gray-100 hover:text-gray-900"
+                >
+                  {{ item.toUpperCase() }}
+                </a>
               </div>
             </div>
           </div>