v0.1.12 with JSON sampling

Author: pdesmarets

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "Cassandra",
-	"version": "0.1.11",
-	"versionDate": "2018-08-18",
+	"version": "0.1.12",
+	"versionDate": "2018-08-22",
 	"author": "hackolade",
 	"engines": {
 		"hackolade": "2.1.x",

reverse_engineering/api.js

@@ -4,7 +4,6 @@ const async = require('async');
 const cassandra = require('./cassandraHelper');
 const systemKeyspaces = require('./package').systemKeyspaces;
 
-
 module.exports = {
 	connect: function(connectionInfo, logger, cb){
 		logger.clear();
@@ -53,9 +52,7 @@ module.exports = {
 
 		const tables = data.collectionData.collections;
 		const keyspacesNames = data.collectionData.dataBaseNames;
-		const fieldInference = data.fieldInference;
 		const includeEmptyCollection = data.includeEmptyCollection;
-		const includeSystemCollection = data.includeSystemCollection;
 		const recordSamplingSettings = data.recordSamplingSettings;
 
 		async.map(keyspacesNames, (keyspaceName, keyspaceCallback) => {
@@ -110,11 +107,11 @@ module.exports = {
 							return packageData;
 						})
 						.then(packageData => {
-							return packageData && columns && columns.length ? cassandra.scanRecords(keyspaceName, tableName) : null;
+							return packageData && columns && columns.length ? cassandra.scanRecords(keyspaceName, tableName, recordSamplingSettings) : null;
 						})
 						.then(columns => {
 							if (columns) {
-								packageData.documents = columns.rows;
+								packageData.documents = columns;
 							}
 							return tableCallback(null, packageData);
 						})

reverse_engineering/cassandraHelper.js

@@ -20,6 +20,7 @@ const connect = (info) => {
 
 const close = () => {
 	if (state.client) {
+		state.client.shutdown();
 		state.client = null;
 	}
 };
@@ -73,7 +74,7 @@ const getColumnInfo = (keyspace, table) => {
 };
 
 const prepareConnectionDataItem = (keyspace, tables) => {
-	let connectionDataItem = {
+	const connectionDataItem = {
 		dbName: keyspace,
 		dbCollections: tables
 	};
@@ -93,10 +94,24 @@ const getTableSchema = (columns, udtHash) => {
 	return { properties: schema };
 };
 
-const scanRecords = (keyspace, table) => {
-	const options = { prepare : true , fetchSize : 1000 };
-	const query = `SELECT * FROM "${keyspace}"."${table}"`;
-	return execute(query);
+const scanRecords = (keyspace, table, recordSamplingSettings) => {
+	const defaultCount = 1000;
+	const query = `SELECT COUNT(*) FROM "${keyspace}"."${table}"`;
+	
+	return execute(query)
+	.then(count => new Promise((resolve, reject) => {
+		const rowsCount = _.get(count, 'rows[0].count.low', defaultCount);
+		const size = getSampleDocSize(rowsCount, recordSamplingSettings)
+		const options = { prepare : true , autoPage: true };
+		const selQuery = `SELECT * FROM "${keyspace}"."${table}" LIMIT ${size}`;
+		let rows = [];
+		
+		state.client.eachRow(selQuery, [], options, function(n, row) {
+			rows.push(row)
+		}, (err, rs) => {
+			return (err ? reject(err) : resolve(rows))
+		});
+	}));
 };
 
 
@@ -148,7 +163,7 @@ const handlePartitionKeys = (partitionKeys) => {
 
 const handleClusteringKeys = (table) => {
 	return (table.clusteringKeys || []).map((item, index) => {
-		let clusteringOrder = table.clusteringOrder ? table.clusteringOrder[index] : '';
+		const clusteringOrder = table.clusteringOrder ? table.clusteringOrder[index] : '';
 		return {
 			name: item.name,
 			type: getKeyOrder(clusteringOrder)
@@ -243,8 +258,6 @@ const handleUDA = (uda) => {
 	return udaData;
 };
 
-
-
 const getPackageData = (data, includeEmptyCollection) => {
 	let packageData = {
 		dbName: data.keyspaceName,
@@ -280,16 +293,14 @@ const filterKeyspaces = (keyspaces, systemKeyspaces) => {
 	return _.difference(keyspaces, systemKeyspaces || []);
 };
 
-/*
+
 const getSampleDocSize = (count, recordSamplingSettings) => {
-	let per = recordSamplingSettings.relative.value;
+	const per = recordSamplingSettings.relative.value;
 	return (recordSamplingSettings.active === 'absolute')
 		? recordSamplingSettings.absolute.value
 			: Math.round( count/100 * per);
 };
 
-*/
-
 module.exports = {
 	connect,
 	close,