Merge remote-tracking branch 'origin/release'

README.md

@@ -8,13 +8,13 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/8c98a251ca64daf434f2/maintainability)](https://codeclimate.com/github/huridocs/uwazi/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/8c98a251ca64daf434f2/test_coverage)](https://codeclimate.com/github/huridocs/uwazi/test_coverage)
 
-Uwazi is an open-source document manager.
+Uwazi is a flexible database application to capture and organise collections of information with a particular focus on document management. HURIDOCS started Uwazi and is supporting dozens of human rights organisations globally to use the tool.
 
 [Uwazi](https://www.uwazi.io/) | [HURIDOCS](https://huridocs.org/)
 
 Read the [user guide](https://github.com/huridocs/uwazi/wiki)
 
-# Intallation guide
+# Installation guide
 
 - [Dependencies](#dependencies)
 - [Production](#production)

app/api/documents/specs/documents.spec.ts

@@ -4,7 +4,7 @@ import { catchErrors } from 'api/utils/jasmineHelpers';
 import { mockID } from 'shared/uniqueID';
 import relationships from 'api/relationships';
 import entities from 'api/entities';
-import search from 'api/search/search';
+import { search } from 'api/search';
 import db from 'api/utils/testing_db';
 import { files } from 'api/files';
 import { fileExists, uploadsPath } from 'api/files/filesystem';

app/api/entities/entities.js

@@ -5,7 +5,7 @@ import ID from 'shared/uniqueID';
 import { propertyTypes } from 'shared/propertyTypes';
 import date from 'api/utils/date';
 import relationships from 'api/relationships/relationships';
-import search from 'api/search/search';
+import { search } from 'api/search';
 import templates from 'api/templates/templates';
 import translationsModel from 'api/i18n/translations';
 import path from 'path';
@@ -357,7 +357,7 @@ export default {
   },
 
   /** Bulk rebuild relationship-based metadata objects as {value = id, label: title}. */
-  async bulkUpdateMetadataFromRelationships(query, language, limit = 200) {
+  async bulkUpdateMetadataFromRelationships(query, language, limit = 200, reindex = true) {
     const process = async (offset, totalRows) => {
       if (offset >= totalRows) {
         return;
@@ -366,7 +366,8 @@ export default {
       const entities = await this.get(query, 'sharedId', { skip: offset, limit });
       await this.updateMetdataFromRelationships(
         entities.map(entity => entity.sharedId),
-        language
+        language,
+        reindex
       );
       await process(offset + limit, totalRows);
     };
@@ -474,7 +475,7 @@ export default {
   },
 
   /** Rebuild relationship-based metadata objects as {value = id, label: title}. */
-  async updateMetdataFromRelationships(entities, language) {
+  async updateMetdataFromRelationships(entities, language, reindex = true) {
     const entitiesToReindex = [];
     const _templates = await templates.get();
     await Promise.all(
@@ -507,11 +508,14 @@ export default {
         }
       })
     );
-    await search.indexEntities({ sharedId: { $in: entitiesToReindex } });
+
+    if (reindex) {
+      await search.indexEntities({ sharedId: { $in: entitiesToReindex } });
+    }
   },
 
   /** Handle property deletion and renames. */
-  async updateMetadataProperties(template, currentTemplate, language) {
+  async updateMetadataProperties(template, currentTemplate, language, reindex = true) {
     const actions = { $rename: {}, $unset: {} };
     template.properties = await generateNamesAndIds(template.properties);
     template.properties.forEach(property => {
@@ -540,10 +544,16 @@ export default {
       await model.updateMany({ template: template._id }, actions);
     }
 
-    if (!template.properties.find(p => p.type === propertyTypes.relationship)) {
+    if (!template.properties.find(p => p.type === propertyTypes.relationship) && reindex) {
       return search.indexEntities({ template: template._id });
     }
-    return this.bulkUpdateMetadataFromRelationships({ template: template._id, language }, language);
+
+    return this.bulkUpdateMetadataFromRelationships(
+      { template: template._id, language },
+      language,
+      200,
+      reindex
+    );
   },
 
   async deleteFiles(deletedDocs) {

app/api/entities/specs/entities.spec.js

@@ -7,7 +7,7 @@ import db from 'api/utils/testing_db';
 import entitiesModel from 'api/entities/entitiesModel';
 import fs from 'fs';
 import relationships from 'api/relationships';
-import search from 'api/search/search';
+import { search } from 'api/search';
 import { uploadsPath } from 'api/files/filesystem';
 
 import entities from '../entities.js';
@@ -306,7 +306,6 @@ describe('entities', () => {
         expect(relatedEntity.metadata.enemies[1].label).toBe('translated2');
       });
 
-      /*eslint-disable */
       it('should index entities changed after propagating label change', async () => {
         const doc = {
           _id: shared2,
@@ -1017,7 +1016,7 @@ describe('entities', () => {
 
     describe('when database deletion throws an error', () => {
       it('should reindex the documents', async () => {
-        spyOn(entitiesModel, 'delete').and.callFake(() => Promise.reject('error'));
+        spyOn(entitiesModel, 'delete').and.callFake(() => Promise.reject(new Error('error')));
         let error;
         try {
           await entities.delete('shared');

app/api/files/filesystem.ts

@@ -72,7 +72,7 @@ const appendFile = promisify(fs.appendFile);
 
 const fileExists = async (filePath: FilePath): Promise<boolean> =>
   new Promise((resolve, reject) => {
-    fs.stat(filePath, err => {
+    fs.access(filePath, err => {
       if (err === null) {
         resolve(true);
       }

app/api/files/specs/exportRoutes.spec.ts

@@ -1,7 +1,7 @@
 import { Writable } from 'stream';
 import request from 'supertest';
 import { setUpApp } from 'api/utils/testingRoutes';
-import search from 'api/search/search';
+import { search } from 'api/search';
 import db from 'api/utils/testing_db';
 import csvExporter from 'api/csv/csvExporter';
 import * as filesystem from 'api/files/filesystem';

app/api/files/specs/filesystem.spec.js

@@ -9,6 +9,11 @@ describe('files', () => {
     fs.writeFileSync(`${__dirname}/file2`, '');
   });
 
+  afterAll(() => {
+    fs.unlinkSync(`${__dirname}/file1`);
+    fs.unlinkSync(`${__dirname}/file2`);
+  });
+
   describe('deleteFiles', () => {
     it('should delete all files passed', async () => {
       await deleteFiles([`${__dirname}/file1`, `${__dirname}/file2`]);

app/api/files/specs/jsRoutes.spec.js

@@ -5,7 +5,7 @@ import { catchErrors } from 'api/utils/jasmineHelpers';
 import db from 'api/utils/testing_db';
 import entities from 'api/entities';
 import { settingsModel } from 'api/settings/settingsModel';
-import search from 'api/search/search';
+import { search } from 'api/search';
 import request from 'supertest';
 import express from 'express';
 

app/api/files/specs/publicRoutes.spec.ts

@@ -41,7 +41,7 @@ describe('public routes', () => {
   beforeEach(async () => {
     spyOn(search, 'indexEntities').and.returnValue(Promise.resolve());
     spyOn(Date, 'now').and.returnValue(1000);
-    spyOn(errorLog, 'error'); //just to avoid annoying console output
+    spyOn(errorLog, 'error');
     await db.clearAllAndLoad(fixtures);
     setupTestUploadedPaths();
   });
@@ -65,15 +65,15 @@ describe('public routes', () => {
       );
 
       const [newEntity] = await entities.get({ title: 'public submit' });
-      const attachments = newEntity.attachments || [];
+      const { attachments } = newEntity;
       expect(attachments).toEqual([expect.objectContaining({ originalname: 'attachment.txt' })]);
 
       const [uploadedFile] = await files.get({ entity: newEntity.sharedId });
       expect(uploadedFile.originalname).toBe('12345.test.pdf');
       expect(uploadedFile.status).toBe('ready');
 
       expect(await fileExists(uploadsPath(uploadedFile.filename))).toBe(true);
-      expect(await fileExists(attachmentsPath(attachments[0].filename))).toBe(true);
+      expect(await fileExists(attachmentsPath(attachments?.[0].filename))).toBe(true);
     });
 
     it('should send an email', async () => {

app/api/odm/specs/model.spec.ts

@@ -43,7 +43,8 @@ describe('ODM Model', () => {
   };
 
   it('should register all the models to the requirable models hashmap', () => {
-    expect(models).toEqual({});
+    expect(models.tempSchema).toBeUndefined();
+    expect(models.anotherSchema).toBeUndefined();
     const model1 = instanceTestingModel('tempSchema', testSchema);
     const model2 = instanceTestingModel('anotherSchema', new mongoose.Schema({ name: String }));
 

app/api/relationships/relationships.js

@@ -7,7 +7,7 @@ import { generateID } from 'api/odm';
 import { createError } from 'api/utils';
 
 import model from './model';
-import search from '../search/search';
+import { search } from '../search';
 import { generateNamesAndIds } from '../templates/utils';
 
 import { filterRelevantRelationships, groupRelationships } from './groupByRelationships';

app/api/relationships/specs/relationships.spec.js

@@ -29,7 +29,7 @@ import fixtures, {
   template,
 } from './fixtures';
 import relationships from '../relationships';
-import search from '../../search/search';
+import { search } from '../../search';
 
 describe('relationships', () => {
   beforeEach(done => {

app/api/search/deprecatedRoutes.js

@@ -1,9 +1,8 @@
 import Joi from 'joi';
 import entities from 'api/entities';
 import { searchSchema } from 'api/search/searchSchema';
-import search from './search';
+import { search } from './search';
 import { validation, parseQuery } from '../utils';
-import needsAuthorization from '../auth/authMiddleware';
 
 export default app => {
   app.get(
@@ -61,11 +60,4 @@ export default app => {
         .then(results => res.json(results))
         .catch(next)
   );
-
-  app.get('/api/search/unpublished', needsAuthorization(['admin', 'editor']), (req, res, next) => {
-    search
-      .getUploadsByUser(req.user, req.language)
-      .then(response => res.json({ rows: response }))
-      .catch(next);
-  });
 };

app/api/search/elastic.ts

@@ -0,0 +1,86 @@
+import elasticSearch, { RequestParams } from '@elastic/elasticsearch';
+import {
+  TransportRequestOptions,
+  RequestBody,
+  RequestNDBody,
+} from '@elastic/elasticsearch/lib/Transport';
+import { tenants } from 'api/tenants';
+import { EntitySchema } from 'shared/types/entityType';
+import { SearchResponse, IndicesPutMapping, IndicesDelete, IndicesCreate } from './elasticTypes';
+
+const elasticClient = new elasticSearch.Client({
+  node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200',
+});
+
+const elastic = {
+  async search(params?: RequestParams.Search<RequestBody>, options?: TransportRequestOptions) {
+    return elasticClient.search<SearchResponse<EntitySchema>, RequestBody>(
+      { ...params, index: tenants.current().indexName },
+      options
+    );
+  },
+
+  async delete(params: RequestParams.Delete, options?: TransportRequestOptions) {
+    return elasticClient.delete({ ...params, index: tenants.current().indexName }, options);
+  },
+
+  async bulk(params: RequestParams.Bulk<RequestNDBody>, options?: TransportRequestOptions) {
+    return elasticClient.bulk({ ...params, index: tenants.current().indexName }, options);
+  },
+
+  async deleteByQuery(
+    params: RequestParams.DeleteByQuery<RequestBody>,
+    options?: TransportRequestOptions
+  ) {
+    return elasticClient.deleteByQuery({ ...params, index: tenants.current().indexName }, options);
+  },
+
+  indices: {
+    async putMapping(params: IndicesPutMapping, options?: TransportRequestOptions) {
+      return elasticClient.indices.putMapping(
+        { ...params, index: tenants.current().indexName },
+        options
+      );
+    },
+
+    async getMapping(params?: RequestParams.IndicesGetMapping, options?: TransportRequestOptions) {
+      return elasticClient.indices.getMapping(
+        { ...params, index: tenants.current().indexName },
+        options
+      );
+    },
+
+    async delete(params?: IndicesDelete, options?: TransportRequestOptions) {
+      return elasticClient.indices.delete(
+        { ...params, index: tenants.current().indexName },
+        options
+      );
+    },
+
+    async create(params?: IndicesCreate, options?: TransportRequestOptions) {
+      return elasticClient.indices.create(
+        { ...params, index: tenants.current().indexName },
+        options
+      );
+    },
+
+    async refresh(params?: RequestParams.IndicesRefresh, options?: TransportRequestOptions) {
+      return elasticClient.indices.refresh(
+        { ...params, index: tenants.current().indexName },
+        options
+      );
+    },
+
+    async validateQuery(
+      params?: RequestParams.IndicesValidateQuery<RequestBody>,
+      options?: TransportRequestOptions
+    ) {
+      return elasticClient.indices.validateQuery(
+        { ...params, index: tenants.current().indexName },
+        options
+      );
+    },
+  },
+};
+
+export { elastic, elasticClient };

app/api/search/elasticTypes.ts

@@ -0,0 +1,50 @@
+import { RequestParams } from '@elastic/elasticsearch';
+import { RequestBody } from '@elastic/elasticsearch/lib/Transport';
+
+interface ShardsResponse {
+  total: number;
+  successful: number;
+  failed: number;
+  skipped: number;
+}
+
+interface Explanation {
+  value: number;
+  description: string;
+  details: Explanation[];
+}
+
+export interface SearchResponse<T> {
+  took: number;
+  // eslint-disable-next-line camelcase
+  timed_out: boolean;
+  // eslint-disable-next-line camelcase
+  _scroll_id?: string;
+  _shards: ShardsResponse;
+  hits: {
+    total: number;
+    // eslint-disable-next-line camelcase
+    max_score: number;
+    hits: Array<{
+      _index: string;
+      _type: string;
+      _id: string;
+      _score: number;
+      _source: T;
+      _version?: number;
+      _explanation?: Explanation;
+      fields?: any;
+      highlight?: any;
+      // eslint-disable-next-line camelcase
+      inner_hits?: any;
+      // eslint-disable-next-line camelcase
+      matched_queries?: string[];
+      sort?: string[];
+    }>;
+  };
+  aggregations?: any;
+}
+
+export type IndicesDelete = Omit<RequestParams.IndicesDelete, 'index'>;
+export type IndicesCreate = Omit<RequestParams.IndicesCreate<RequestBody>, 'index'>;
+export type IndicesPutMapping = Omit<RequestParams.IndicesPutMapping<RequestBody>, 'index'>;