Skip to content

Commit 5116140

Browse files
mtseluikomtseluiko
authored
Merge pull request #106 from DrMyroslav/HCK-2717-solr-search-index-how-to-determine-the-profile-value
Hck 2717 solr search index how to determine the profile value
2 parents fff54d8 + 8e28078 commit 5116140

File tree

8 files changed

+145
-41
lines changed

8 files changed

+145
-41
lines changed

‎forward_engineering/api.js

+4-2
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ module.exports = {
9999
const UDF = getUserDefinedFunctions(retrieveUDF(containerData));
100100
const UDA = getUserDefinedAggregations(retrieveUDA(containerData));
101101

102+
const dbVersion = data.modelData[0].dbVersion;
103+
102104
cqlScriptData.push(
103105
keyspace,
104106
...generalUDT
@@ -140,13 +142,13 @@ module.exports = {
140142
isKeyspaceActivated,
141143
});
142144
const indexes = getIndexes(
143-
retrieveIndexes(entityData),
145+
retrieveIndexes(entityData, dbVersion),
144146
dataSources,
145147
entityName,
146148
containerName,
147149
isEntityActivated,
148150
isKeyspaceActivated,
149-
data.modelData[0].dbVersion
151+
dbVersion
150152
);
151153

152154
cqlScriptData.push(...internalUdt, table, indexes);

‎forward_engineering/helpers/createHelper.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ const getCreateTableScript = (data, isKeyspaceActivated) => {
2727
udtTypeMap,
2828
isKeyspaceActivated: isEntityChildrenActivated
2929
});
30-
const indexes = getIndexes(retrieveIndexes(data.entityData), dataSources, entityName, containerName, isEntityActivated, isKeyspaceActivated);
30+
const dbVersion = data.modelData?.[0]?.dbVersion;
31+
const indexes = getIndexes(retrieveIndexes(data.entityData, dbVersion), dataSources, entityName, containerName, isEntityActivated, isKeyspaceActivated, dbVersion);
3132
const UDF = getUserDefinedFunctions(retrieveUDF(data.containerData));
3233
const UDA = getUserDefinedAggregations(retrieveUDA(data.containerData));
3334

‎forward_engineering/helpers/generalHelper.js

+27-8
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,16 @@ const fs = require('fs');
33
const path = require('path');
44
const TAB_SIZE = 2;
55

6+
const getDataStaxVersion = (dbVersion = '') => {
7+
if (!dbVersion.startsWith('DSE')) {
8+
return '';
9+
}
10+
11+
const version = dbVersion.split(' ')[1] || '';
12+
13+
return version.split('.')[0] || '';
14+
};
15+
616
const tab = (text, count = 1) => {
717
const space = ' '.repeat(count * TAB_SIZE);
818

@@ -21,7 +31,7 @@ const retrieveEntityName = (entityConfig) => retrivePropertyFromConfig(
2131
);
2232
const retrieveUDF = (containerConfig) => retrivePropertyFromConfig(containerConfig, 1, "UDFs", []);
2333
const retrieveUDA = (containerConfig) => retrivePropertyFromConfig(containerConfig, 2, "UDAs", []);
24-
const retrieveIndexes = (entityConfig) => {
34+
const retrieveIndexes = (entityConfig, dbVersion) => {
2535
const indexTab = entityConfig[1];
2636
const result = {
2737
indexes: retrivePropertyFromConfig(entityConfig, 1, "SecIndxs", []),
@@ -32,24 +42,33 @@ const retrieveIndexes = (entityConfig) => {
3242
indexType: 'search',
3343
columns: indexTab.searchIndexColumns,
3444
config: indexTab.searchIndexConfig,
35-
profiles: getIndexProfiles(indexTab.searchIndexProfiles),
45+
profiles: getIndexProfiles(indexTab.searchIndexProfiles, dbVersion),
3646
options: indexTab.searchIndexOptions,
3747
ifNotExist: indexTab.searchIndexIfNotExist
3848
};
3949
}
4050

4151
return result;
4252
};
43-
const getIndexProfiles = (searchIndexProfiles) => {
44-
if (!Array.isArray(searchIndexProfiles)) {
45-
return [searchIndexProfiles].filter(Boolean);
46-
}
53+
54+
const filterSpaceSaving = (searchIndexProfiles, profileForFilter) => searchIndexProfiles.filter(profile => !profileForFilter.includes(profile));
55+
56+
const getIndexProfiles = (searchIndexProfiles, dbVersion) => {
57+
const dataStaxVersion = getDataStaxVersion(dbVersion);
58+
const spaceSavingNoTextfieldNotExist = dataStaxVersion.length ? Number(dataStaxVersion) >=6 : false;
59+
const profileForFilter = [spaceSavingNoTextfieldNotExist ? "spaceSavingNoTextfield" : ''];
60+
61+
searchIndexProfiles = !Array.isArray(searchIndexProfiles) ? [searchIndexProfiles].filter(Boolean) : searchIndexProfiles;
4762

48-
const isSpaceSavingAll = [
63+
searchIndexProfiles = filterSpaceSaving(searchIndexProfiles, profileForFilter);
64+
65+
const spaceSavingAll = filterSpaceSaving([
4966
"spaceSavingNoJoin",
5067
"spaceSavingNoTextfield",
5168
"spaceSavingSlowTriePrecision"
52-
].every(item => searchIndexProfiles.includes(item));
69+
], profileForFilter);
70+
71+
const isSpaceSavingAll = spaceSavingAll.every(item => searchIndexProfiles.includes(item));
5372

5473
if (isSpaceSavingAll) {
5574
return ['spaceSavingAll'];

‎forward_engineering/helpers/schemaHelper.js

+12-9
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@
22

33
const attributesForReturn = ['isActivated', 'type', 'compositePartitionKey'];
44

5-
const getAttributes = (field = {}) => {
5+
const getAttributes = (field = {}, allAttributes = false) => {
6+
if (allAttributes) {
7+
return field;
8+
}
69
return attributesForReturn.reduce((fieldAttributes, attribute) => {
710
if (!field.hasOwnProperty(attribute) || field[attribute] === undefined) {
811
return fieldAttributes
@@ -44,30 +47,30 @@ const getPathById = (schema, id, path) => {
4447
}
4548
};
4649

47-
const getRootItemMetadataById = (id, properties) => {
50+
const getRootItemMetadataById = (id, properties, allAttributes) => {
4851
const propertyName = Object.keys(properties).find(propertyName => (properties[propertyName].GUID === id || properties[propertyName].id === id));
4952
const propertyValue = properties[propertyName];
5053
if (propertyValue && (propertyValue.code || propertyValue.name)) {
51-
return { name: propertyValue.code || propertyValue.name, ...getAttributes(properties[propertyName]) };
54+
return { name: propertyValue.code || propertyValue.name, ...getAttributes(properties[propertyName], allAttributes) };
5255
}
5356

54-
return { name: propertyName, ...getAttributes(properties[propertyName]) };
57+
return { name: propertyName, ...getAttributes(properties[propertyName], allAttributes) };
5558
};
5659

57-
const findFieldMetadataById = (id, source) => {
60+
const findFieldMetadataById = (id, source, allAttributes) => {
5861
let path = getPathById(source, id, []);
5962

6063
if (path) {
61-
return getRootItemMetadataById(path[0], source.properties);
64+
return getRootItemMetadataById(path[0], source.properties, allAttributes);
6265
} else {
6366
return { name: "" };
6467
}
6568
};
6669

67-
const getAttributesDataByIds = (ids, sources) => {
70+
const getAttributesDataByIds = (ids, sources, allAttributes) => {
6871
return ids.reduce((hash, id) => {
6972
for (let i = 0; i < sources.length; i++) {
70-
const fieldData = findFieldMetadataById(id, sources[i]);
73+
const fieldData = findFieldMetadataById(id, sources[i], allAttributes);
7174

7275
if (fieldData?.name) {
7376
return Object.assign({}, hash, { [id]: fieldData});
@@ -80,5 +83,5 @@ const getAttributesDataByIds = (ids, sources) => {
8083

8184
module.exports = {
8285
getPathById,
83-
getNamesByIds: getAttributesDataByIds
86+
getNamesByIds: getAttributesDataByIds,
8487
};

‎forward_engineering/helpers/updateHelper.js

+6-2
Original file line numberDiff line numberDiff line change
@@ -126,9 +126,13 @@ const getIsColumnInIndex = (item, columnName, data) => {
126126

127127
const dataSources = [itemData, data.modelDefinitions];
128128
const secIndexes = _.get(item, 'role.SecIndxs', [])
129-
.map(index => getDataColumnIndex(dataSources, {}, index, 'SecIndxKey')).map(index => index.name).filter(Boolean);
129+
.map(index => getDataColumnIndex({ dataSources, idToNameHashTable: {}, column: index, key: 'SecIndxKey' }))
130+
.map(index => index.name)
131+
.filter(Boolean);
130132
const searchIndexes = _.get(item, 'role.searchIndexColumns', [])
131-
.map(index => getDataColumnIndex(dataSources, {}, index)).map(index => index.name).filter(Boolean);
133+
.map(index => getDataColumnIndex({ dataSources, idToNameHashTable: {}, column: index }))
134+
.map(index => index.name)
135+
.filter(Boolean);
132136
return [...searchIndexes, ...secIndexes].includes(columnName);
133137
};
134138

‎forward_engineering/helpers/updateHelpers/indexHelper.js

+40-18
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ const {
77
getDiffConfig,
88
getDiffIndexProfiles,
99
isEqualIndex,
10+
prepareSearchIndexProfile,
1011
} = require('./indexService');
1112
let _;
1213

@@ -36,23 +37,23 @@ const setNameCollectionsScript = (keyspaceName, name, type) => {
3637

3738
const indexSearchProperties = ['searchIndexProfiles', 'searchIndexOptions'];
3839

39-
const getDataSearchIndex = indexTab => {
40+
const getDataSearchIndex = (indexTab, dbVersion) => {
4041
return {
4142
indexType: 'search',
4243
columns: indexTab.searchIndexColumns,
4344
config: indexTab.searchIndexConfig,
44-
profiles: getIndexProfiles(indexTab.searchIndexProfiles),
45+
profiles: getIndexProfiles(indexTab.searchIndexProfiles, dbVersion),
4546
options: indexTab.searchIndexOptions,
4647
ifNotExist: indexTab.searchIndexIfNotExist
4748
};
4849
}
4950

50-
const getModifyDataSearchIndex = (item, type) => {
51+
const getModifyDataSearchIndex = (item, type, dbVersion) => {
5152
const itemData = [...indexSearchProperties, 'searchIndexIfNotExist', 'searchIndexColumns', 'searchIndexConfig']
5253
.map(property => ([property, (item?.compMod || {})[property]?.[type]]))
5354
.filter(([__, value]) => !!value);
5455
item = { ...item, ...Object.fromEntries(itemData) };
55-
return getDataSearchIndex(item);
56+
return getDataSearchIndex(item, dbVersion);
5657
}
5758

5859
const getDataForScript = (newData, oldData, isEqual = _.isEqual) => {
@@ -78,13 +79,26 @@ const getDataForScript = (newData, oldData, isEqual = _.isEqual) => {
7879
}
7980
}
8081

81-
const getDataForSearchIndexScript = role => {
82+
const getDataForSearchIndexScript = (role, dataSources, dbVersion) => {
8283
const { compMod } = role;
83-
const dataSearchIndex = getDataSearchIndex(role);
84+
const oldIdToNameHashTable = _.get(role, 'compMod.oldIdToNameHashTable', {});
85+
const columns = _.get(role, 'compMod.searchIndexColumns', {});
86+
const filterColumn = column => column?.name && !column?.compositePartitionKey;
87+
const oldColumns = (columns.old || [])
88+
.map(column => getDataColumnIndex({
89+
dataSources,
90+
column,
91+
allAttributes: true,
92+
idToNameHashTable: oldIdToNameHashTable,
93+
}))
94+
.filter(filterColumn);
95+
const dataSearchIndex = getDataSearchIndex(role, dbVersion);
8496
const searchIndex = _.get(role, 'compMod.searchIndex', {});
97+
const newProfiles = compMod?.searchIndexProfiles?.new;
98+
const oldProfiles = prepareSearchIndexProfile(compMod?.searchIndexProfiles?.old, newProfiles, oldColumns)
8599
const searchPropertiesCompare = _.merge(
86100
getDiffOptions(compMod?.searchIndexOptions?.old, compMod?.searchIndexOptions?.new),
87-
getDiffIndexProfiles(compMod?.searchIndexProfiles?.old, compMod?.searchIndexProfiles?.new),
101+
getDiffIndexProfiles(oldProfiles, newProfiles),
88102
);
89103

90104
let dropData;
@@ -95,11 +109,11 @@ const getDataForSearchIndexScript = role => {
95109
}
96110

97111
if (!searchIndex.old) {
98-
addData = getModifyDataSearchIndex(role, 'new');
112+
addData = getModifyDataSearchIndex(role, 'new', dbVersion);
99113
} else if (!searchIndex.new) {
100114
dropData = dataSearchIndex;
101115
} else if (!_.isEmpty(searchPropertiesCompare.modifyData) || !_.isEmpty(searchPropertiesCompare.dropData)) {
102-
addData = getModifyDataSearchIndex(role, 'new');
116+
addData = getModifyDataSearchIndex(role, 'new', dbVersion);
103117
dropData = dataSearchIndex;
104118
}
105119
return {
@@ -108,8 +122,8 @@ const getDataForSearchIndexScript = role => {
108122
};
109123
}
110124

111-
const getFieldDataByKeyId = (dataSources, idToNameHashTable, keyId) => {
112-
const fieldData = getNamesByIds([keyId], dataSources)[keyId] || {};
125+
const getFieldDataByKeyId = ({ dataSources, idToNameHashTable, keyId, allAttributes = false }) => {
126+
const fieldData = getNamesByIds([keyId], dataSources, allAttributes)[keyId] || {};
113127
if (fieldData.name) {
114128
return fieldData
115129
}
@@ -119,10 +133,16 @@ const getFieldDataByKeyId = (dataSources, idToNameHashTable, keyId) => {
119133
return { name };
120134
}
121135

122-
const getDataColumnIndex = (dataSources, idToNameHashTable, column = {}, key = 'key') => {
136+
const getDataColumnIndex = ({
137+
dataSources,
138+
idToNameHashTable,
139+
column = {},
140+
key = 'key',
141+
allAttributes = false,
142+
}) => {
123143
setDependencies(dependencies);
124144
const keyId = _.get(column, `${key}[0].keyId`, '');
125-
const fieldData = getFieldDataByKeyId(dataSources, idToNameHashTable, keyId);
145+
const fieldData = getFieldDataByKeyId({ dataSources, idToNameHashTable, keyId, allAttributes }) || {};
126146

127147
return {
128148
..._.omit(column, key),
@@ -136,9 +156,11 @@ const getDataForSearchIndexColumns = (item, dataSources) => {
136156
const columns = _.get(item, 'role.compMod.searchIndexColumns', {});
137157
const filterColumn = column => column.name && !column.compositePartitionKey;
138158
const newColumns = (columns.new || [])
139-
.map(column => getDataColumnIndex(dataSources, newIdToNameHashTable, column)).filter(filterColumn);
159+
.map(column => getDataColumnIndex({ dataSources, idToNameHashTable: newIdToNameHashTable, column }))
160+
.filter(filterColumn);
140161
const oldColumns = (columns.old || [])
141-
.map(column => getDataColumnIndex(dataSources, oldIdToNameHashTable, column)).filter(filterColumn);
162+
.map(column => getDataColumnIndex({ dataSources, idToNameHashTable: oldIdToNameHashTable, column }))
163+
.filter(filterColumn);
142164
const isEqual = (oldValue, newValue) => oldValue?.name === newValue?.name;
143165
return getDataForScript(newColumns, oldColumns, isEqual);
144166
}
@@ -280,7 +302,7 @@ const getAddIndexScript = data => {
280302
const getCreatedIndex = data => {
281303
const { item, dataSources, tableName, keyspaceName, isActivated, dbVersion } = data;
282304
const isSearchIndex = !!item.role?.searchIndex;
283-
const dataForSearchIndexScript = getDataSearchIndex(item.role || {});
305+
const dataForSearchIndexScript = getDataSearchIndex(item.role || {}, dbVersion);
284306

285307
const script = getIndexes({
286308
searchIndex: isSearchIndex && dataForSearchIndexScript,
@@ -314,7 +336,7 @@ const getDeletedIndex = data => {
314336
const getUpdateSearchIndexScript = data => {
315337
const { item, keyspaceName, tableName, dbVersion, isActivated, dataSources } = data;
316338

317-
const dataForScript = getDataForSearchIndexScript(item.role);
339+
const dataForScript = getDataForSearchIndexScript(item.role, dataSources, dbVersion);
318340

319341
if (dataForScript.dropData || dataForScript.addData) {
320342
const dropIndexSearchScript = getDropSearchIndexScript(
@@ -361,7 +383,7 @@ const prepareIndexes = (idToNameHashTable, dataSources, indexes = []) => {
361383
return indexes.map(index => {
362384
const secIndexesKey = _.get(index, 'SecIndxKey', []).map(key => ({
363385
...key,
364-
name: getFieldDataByKeyId(dataSources, idToNameHashTable, _.get(key, 'keyId'))?.name || '',
386+
name: getFieldDataByKeyId({ dataSources, idToNameHashTable, keyId: _.get(key, 'keyId') })?.name || '',
365387
})
366388
);
367389
return {

‎forward_engineering/helpers/updateHelpers/indexService.js

+29
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,16 @@ const INDEX_CONFIG_DEFAULT = {
2727
indexIfNotExist: false,
2828
};
2929

30+
const SEARCH_INDEX_PROFILES_DATA_FOR_PREPARE = {
31+
spaceSavingNoTextfield: {
32+
mode: ['text', 'varchar'],
33+
},
34+
spaceSavingSlowTriePrecision: {
35+
childType: ['timestamp', 'date', 'time', 'numeric', 'timeuuid'],
36+
type: ['timestamp', 'date', 'time', 'numeric', 'timeuuid'],
37+
},
38+
};
39+
3040
const isDiff = (oldValue, newValue) => !_.isEqual(oldValue, newValue);
3141

3242
const getModifiedProperties = (oldProperties, newProperties) => {
@@ -114,10 +124,29 @@ const isEqualIndex = (defaultData, redundantProperty) => (oldData = {}, newData
114124
isEqual && _.isEqual(newData[key], oldData[key]) ? isEqual : false, true);
115125
};
116126

127+
const prepareSearchIndexProfile = (oldProfiles = [], newProfiles = [], oldColumns = []) => {
128+
return Object.entries(SEARCH_INDEX_PROFILES_DATA_FOR_PREPARE).reduce((oldProfiles, [profile, profileData]) => {
129+
if (!newProfiles.includes(profile) || oldProfiles.includes(profile)) {
130+
return oldProfiles;
131+
}
132+
133+
const profileIsNotExist = Object.entries(profileData).reduce((profileValue, [key, keyValues]) => {
134+
if (profileValue) {
135+
return profileValue;
136+
}
137+
const keysOldColumns = oldColumns.map(column => column[key]).filter(Boolean);
138+
return keyValues.some(value => keysOldColumns.includes(value));
139+
}, false);
140+
141+
return profileIsNotExist ? oldProfiles: [...oldProfiles, profile];
142+
}, oldProfiles);
143+
};
144+
117145
module.exports = {
118146
getDiffOptions: getDiffOptions(SEARCH_INDEX_OPTIONS_DEFAULT),
119147
getDiffConfig: getDiff(SEARCH_INDEX_CONFIG_DEFAULT),
120148
getDiffIndexProfiles: getDiff({}),
121149
isEqualIndex: isEqualIndex(INDEX_CONFIG_DEFAULT, [...REDUNDANT_OPTIONS, ...REDUNDANT_PROPERTIES_FOR_INDEX]),
150+
prepareSearchIndexProfile,
122151
REDUNDANT_OPTIONS,
123152
}

0 commit comments

Comments
 (0)