🦢 Switch language package (#296)

* Replace ngram package

* Remove fallback to metadata language

* Add language predition to the update mapping

* Changelog and version bump

CHANGELOG.md

@@ -1,3 +1,11 @@
+# 3.3.0
+
+## Schema
+- `orionLanguage` property has been added.
+
+## Mappings 
+- Language detection is used to populate new property on video update and creation.
+
 # 3.2.1
 
 ## Misc

package.json

@@ -1,6 +1,6 @@
 {
   "name": "orion",
-  "version": "3.2.1",
+  "version": "3.3.0",
   "engines": {
     "node": ">=16"
   },
@@ -76,14 +76,14 @@
     "graphql-tools": "^8.3.11",
     "handlebars": "^4.7.7",
     "haversine-distance": "^1.2.1",
-    "languagedetect": "^2.0.0",
     "lodash": "^4.17.21",
     "node-cache": "^5.1.2",
     "node-schedule": "^2.1.1",
     "p-limit": "3.1.0",
     "patch-package": "^6.5.0",
     "pg": "8.8.0",
     "swagger-ui-express": "^4.6.2",
+    "tinyld": "^1.3.4",
     "type-graphql": "^1.2.0-rc.1",
     "typeorm": "^0.3.11",
     "ua-parser-js": "^1.0.34",

src/mappings/content/video.ts

@@ -116,7 +116,7 @@ export async function processVideoCreatedEvent({
   }
 
   const languageText = [video.title ?? '', video.description ?? ''].join(' ')
-  video.orionLanguage = predictLanguage(languageText) ?? video.language
+  video.orionLanguage = predictLanguage(languageText)
 
   channel.totalVideosCreated += 1
 
@@ -183,6 +183,9 @@ export async function processVideoUpdatedEvent({
     )
   }
 
+  const languageText = [video.title ?? '', video.description ?? ''].join(' ')
+  video.orionLanguage = predictLanguage(languageText)
+
   if (autoIssueNft) {
     await processNft(overlay, block, indexInBlock, extrinsicHash, video, contentActor, autoIssueNft)
   }

src/utils/language.ts

@@ -1,4 +1,4 @@
-import LanguageDetect from 'languagedetect'
+import { detect } from 'tinyld'
 
 function cleanString(input: string): string {
   // Remove symbols, numbers, and emojis
@@ -12,11 +12,8 @@ function cleanString(input: string): string {
   return cleanedString.toLowerCase()
 }
 
-const lngDetector = new LanguageDetect()
-lngDetector.setLanguageType('iso2')
-
 // Example usage
 export const predictLanguage = (text: string): string | undefined => {
   const cleanedText = cleanString(text)
-  return lngDetector.detect(cleanedText, 1)[0]?.[0]
+  return detect(cleanedText) || undefined
 }