Add rules to expand search queries (#121)

* update acknowledgements

* add rules to expand search query

* update version

.github/workflows/coverage.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Set up Node 12
       uses: actions/setup-node@v1
       with:
-        version: 12
+        node-version: 12
     - name: Install dependencies
       run: npm install
     - name: Coverage

.github/workflows/smalltest.yml

@@ -12,7 +12,7 @@ jobs:
     - name: Set up Node 12
       uses: actions/setup-node@v1
       with:
-        version: 12
+        node-version: 12
     - name: Install dependencies
       run: npm install
     - name: Test

.github/workflows/test.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Set up Node ${{ matrix.node }} on ${{ matrix.platform }}
       uses: actions/setup-node@v1
       with:
-        version: ${{ matrix.node }}
+        node-version: ${{ matrix.node }}
     - name: Install dependencies
       run: npm install
     - name: Test

.gitignore

@@ -104,4 +104,5 @@ dist
 .tern-port
 
 /build/
-frontend/
+frontend/
+.vscode

README.md

@@ -82,13 +82,10 @@ Example: `proxy: "http://127.0.0.1:1080"`
 ------------------
 
 <div align=center id="Acknowledgements">
-
-
-
 *Acknowledgements*
 
 *This project does not include any stored or static data, and all the data it presents are collected realtime and automatically from the websites below. Internet data are public but messy and collecting them is a tiring work. Appreciate them for their offering precious and high quality data.*
 
-[JavMost](https://www5.javmost.com)  [AVSOX](https://avsox.net)  [AV女優名 変換君](http://etigoya955.blog49.fc2.com/)  [IndexAV](https://indexav.com)  [JavBus](https://www.javbus.com)  [xopenload.video](https://www.xopenload.video)  [YouAV](https://www.xopenload.video)  [Avgle](https://avgle.com)  [Fembed](https://www.fembed.com) [素人系AV女優大辞典wiki](https://av-help.memo.wiki/) [JavModel](https://javmodel.com/) [Warashi Asian Pornstars Database](http://warashi-asian-pornstars.fr/en/s-0/wapdb-database-of-asian-pornstars-japanese-av-actresses-and-actors) [JavFull.net](https://javfull.net/) [JavLibrary](http://www.javlibrary.com) [XFantasy](https://https://xfantasy.tv)
+[JavMost](https://www5.javmost.com)  [AVSOX](https://avsox.net)  [AV女優名 変換君](http://etigoya955.blog49.fc2.com/)  [IndexAV](https://indexav.com)  [JavBus](https://www.javbus.com)   [YouAV](https://www.xopenload.video)  [Avgle](https://avgle.com) [素人系AV女優大辞典wiki](https://av-help.memo.wiki/) [JavModel](https://javmodel.com/) [Warashi-Asian-Pornstars-Database](http://warashi-asian-pornstars.fr/en/s-0/wapdb-database-of-asian-pornstars-japanese-av-actresses-and-actors) [JavFull](https://javfull.net/) [JavLibrary](http://www.javlibrary.com) [XFantasy](https://https://xfantasy.tv) [JavBus](https://www.javbus.com) [HighPorn](https://highporn.net)
 
 </div>

bin/javpy.js

@@ -12,6 +12,11 @@ const { argv } = yargs
     default: 8081,
     type: 'int',
   })
+  .option('browser', {
+    description: 'open browser automatically',
+    default: true,
+    type: 'bool',
+  })
   .help()
   .alias('help', 'h');
 
@@ -27,4 +32,4 @@ const openBrowser = () => {
   }
 };
 
-server.run(argv.port, openBrowser);
+server.run(argv.port, argv.browser === 'false' ? undefined : openBrowser);

package.json

@@ -1,8 +1,9 @@
 {
   "name": "javpy",
-  "version": "0.7.6",
+  "version": "0.7.7",
   "description": "Enjoy driving on a Javascriptive (originally Pythonic) way to Japanese AV!",
   "scripts": {
+    "dev": "node ./bin/javpy.js --browser=false",
     "test": "mocha ./test/test.js --timeout 60000",
     "coverage": "nyc --reporter=lcov mocha --timeout 60000 && codecov -t ${CODECOV_TOKEN}",
     "build": "pkg . --out-path=./build/",
@@ -49,4 +50,4 @@
   "bin": {
     "javpy": "./bin/javpy.js"
   }
-}
+}

src/functions/getActressProfile.js

@@ -3,7 +3,7 @@ const utils = require('./utils');
 
 const getActressProfile = async (ws, reqId, { actress }) => {
   try {
-    const rsp = sources.warashiAsianPornstarsFr.getActressProfile(actress);
+    const rsp = await sources.warashiAsianPornstarsFr.getActressProfile(actress);
     if (rsp) {
       ws.send(JSON.stringify({ response: rsp, reqId }));
     } else {

src/functions/searchByCode.js

@@ -1,33 +1,50 @@
 const sources = require('../sources');
 const utils = require('./utils');
 
+const cartesian = (...a) => a.reduce((x, y) => x.flatMap((d) => y.map((e) => [d, e].flat())));
+
 const searchByCode = async (ws, reqId, { code }) => {
   let something = false;
-  await Promise.allSettled([
-    sources.javmost,
-    sources.avgle,
-    sources.youav,
-    sources.highporn,
-    sources.javhdporn,
-  ].map((source) => source.searchByCode(code).then(async (rsp) => {
-    if (!rsp) {
-      return;
-    }
 
-    if (Array.isArray(rsp)) {
-      await Promise.allSettled(rsp.map((av) => av.then((response) => {
-        if (response) {
-          something = true;
-          ws.send(JSON.stringify({ response, reqId }));
-        }
-      })));
-    } else {
-      something = true;
-      ws.send(JSON.stringify({ response: rsp, reqId }));
-    }
-  }).catch((err) => {
-    console.error(err.message);
-  })));
+  let targets = [code];
+
+  const expander = utils.expandSearch.getExpander(code);
+  if (expander) {
+    targets = expander(code);
+  }
+
+  await Promise.allSettled(
+    cartesian([
+      sources.javmost,
+      sources.avgle,
+      sources.youav,
+      sources.highporn,
+      sources.javhdporn,
+    ], targets)
+      .map(([source, target]) => [[source, target.toUpperCase()], [source, target.toLowerCase()]])
+      .reduce((input1, input2) => input1.concat(input2))
+      .map(
+        ([source, target]) => source.searchByCode(target).then(async (rsp) => {
+          if (!rsp) {
+            return;
+          }
+
+          if (Array.isArray(rsp)) {
+            await Promise.allSettled(rsp.map((av) => av.then((response) => {
+              if (response) {
+                something = true;
+                ws.send(JSON.stringify({ response, reqId }));
+              }
+            })));
+          } else {
+            something = true;
+            ws.send(JSON.stringify({ response: rsp, reqId }));
+          }
+        }).catch((err) => {
+          console.error(err.message);
+        }),
+      ),
+  );
   if (!something) {
     utils.notFound(ws, reqId);
   }

src/functions/utils.js

@@ -12,7 +12,34 @@ const notFound = (ws, reqId) => {
   ws.send(JSON.stringify({ response: 'not found', reqId }));
 };
 
+const expandSearch = {
+  regexes: [
+    {
+      regex: new RegExp(/^\d+[-_]\d+$/),
+      expander: (code) => [code.replace('-', '_'), code.replace('_', '-')],
+    },
+    {
+      regex: new RegExp(/^RETOMN-\d+$/),
+      expander: (code) => [code, code.slice(2, code.length)],
+    },
+    {
+      regex: new RegExp(/\d+_ppv-.+/),
+      expander: (code) => [code, code.replace(/\d+_ppv-/, '')],
+    },
+  ],
+
+  getExpander(code) {
+    for (let i = 0; i < this.regexes.length; i += 1) {
+      if (code.match(this.regexes[i].regex)) {
+        return this.regexes[i].expander;
+      }
+    }
+    return null;
+  },
+};
+
 module.exports = {
   guessLang,
   notFound,
+  expandSearch,
 };

src/sources/javdb.js

@@ -81,9 +81,9 @@ const searchMagnet = async (code) => {
 
 const getNewlyReleased = async (page) => {
   const censoredUrl = `/?page=${page}&vft=0`;
-  const uncensoredUrl = `/uncensored?page=${page}&vft=0`;
+  // const uncensoredUrl = `/uncensored?page=${page}&vft=0`;
 
-  return (await Promise.allSettled([censoredUrl, uncensoredUrl].map(
+  const responses = await Promise.allSettled([censoredUrl].map(
     (url) => requester.get(url).then((rsp) => {
       const dom = new JSDOM(rsp.data).window.document;
       return [...dom.querySelector('#videos').querySelectorAll('.grid-item')].map((video) => {
@@ -95,7 +95,13 @@ const getNewlyReleased = async (page) => {
         return av;
       });
     }).catch(() => []),
-  ))).reduce((a, b) => a.value.concat(b.value));
+  ));
+
+  if (responses.length > 1) {
+    return responses.reduce((a, b) => a.value.concat(b.value));
+  }
+
+  return responses.value;
 };
 
 module.exports = {

src/sources/xslist.js

@@ -0,0 +1,29 @@
+const Axios = require('axios').default;
+const utils = require('./utils');
+const ds = require('../ds');
+
+const requester = utils.requester('https://api.avgle.com');
+
+const searchByCode = async (code) => {
+  const rsp = await requester.get(`/v1/search/${code}/0?limit=1`);
+  const video = rsp.data.response.videos[0];
+
+  const testVideo = await Axios.get(encodeURI(video.video_url), { maxRedirects: 0 });
+  if (testVideo.status === 301) {
+    return null;
+  }
+
+  if (video.title.toLowerCase().includes(code.toLowerCase())) {
+    const av = new ds.AV();
+    av.title = video.title;
+    av.video_url = video.video_url;
+    av.code = code;
+    av.preview_img_url = video.preview_url;
+    return av;
+  }
+  return null;
+};
+
+module.exports = {
+  searchByCode,
+};