export function properDruidRequesterFactory(options: ProperDruidRequesterOptions): Requester.PlywoodRequester<any> {
var {
druidHost,
retry,
timeout,
verbose,
concurrentLimit,
requestDecorator
} = options;
var druidRequester = druidRequesterFactory({
host: druidHost,
timeout: timeout || 30000,
requestDecorator
});
if (retry) {
druidRequester = helper.retryRequesterFactory({
requester: druidRequester,
retry: retry,
delay: 500,
retryOnTimeout: false
});
}
if (verbose) {
druidRequester = helper.verboseRequesterFactory({
requester: druidRequester
});
}
if (concurrentLimit) {
druidRequester = helper.concurrentLimitRequesterFactory({
requester: druidRequester,
concurrentLimit: concurrentLimit
});
}
return druidRequester;
}
static fromJS(parameters: LinkItemJS, context?: LinkItemContext): LinkItem {
if (!context) throw new Error('LinkItem must have context');
const { dataCubes, visualizations } = context;
var dataCubeName = parameters.dataCube;
var dataCube = helper.find(dataCubes, d => d.name === dataCubeName);
if (!dataCube) throw new Error(`can not find dataCube '${dataCubeName}'`);
var essence = Essence.fromJS(parameters.essence, { dataCube, visualizations }).updateSplitsWithFilter();
return new LinkItem({
name: parameters.name,
title: parameters.title,
description: parameters.description,
group: parameters.group,
dataCube,
essence
});
}
export function properRequesterFactory(options: ProperRequesterOptions): Requester.PlywoodRequester<any> {
var {
type,
host,
retry,
timeout,
verbose,
concurrentLimit
} = options;
var requester: Requester.PlywoodRequester<any>;
switch (type) {
case 'druid':
requester = druidRequesterFactory({
host,
timeout: timeout || 30000,
requestDecorator: options.druidRequestDecorator
});
break;
case 'mysql':
requester = mySqlRequesterFactory({
host,
database: options.database,
user: options.user,
password: options.password
});
break;
case 'postgres':
requester = postgresRequesterFactory({
host,
database: options.database,
user: options.user,
password: options.password
});
break;
default:
throw new Error(`unknown requester type ${type}`);
}
if (retry) {
requester = helper.retryRequesterFactory({
requester: requester,
retry: retry,
delay: 500,
retryOnTimeout: false
});
}
if (verbose) {
requester = helper.verboseRequesterFactory({
requester: requester
});
}
if (concurrentLimit) {
requester = helper.concurrentLimitRequesterFactory({
requester: requester,
concurrentLimit: concurrentLimit
});
}
return requester;
}
public findByName(name: string): LinkItem {
return helper.findByName(this.linkItems, name);
}
export function run() {
var parsed = parseArgs();
if (parsed.argv.original.length === 0 || parsed['help']) {
printUsage();
return;
}
if (parsed['version']) {
printVersion();
return;
}
var verbose: boolean = parsed['verbose'];
// Get allow
var allows: string[] = parsed['allow'] || [];
for (let allow of allows) {
if (!(allow === 'eternity' || allow === 'select')) {
console.log("Unexpected allow", allow);
return;
}
}
// Get forced attribute overrides
var attributeOverrides: AttributeJSs = [];
var forceUnique: string[] = parsed['force-unique'] || [];
for (let attributeName of forceUnique) {
attributeOverrides.push({ name: attributeName, special: 'unique' });
}
var forceHistogram: string[] = parsed['force-histogram'] || [];
for (let attributeName of forceHistogram) {
attributeOverrides.push({ name: attributeName, special: 'histogram' });
}
// Get output
var output: string = (parsed['output'] || 'json').toLowerCase();
if (output !== 'json' && output !== 'csv' && output !== 'tsv' && output !== 'flat') {
console.log(`output must be one of json, csv, tsv, or flat (is ${output}})`);
return;
}
// Get host
var host: string = parsed['druid'] || parsed['host'];
if (!host) {
console.log("must have a host");
return;
}
// Get SQL
var query: string = parsed['query'];
if (query) {
try {
var sqlParse = Expression.parseSQL(query);
} catch (e) {
console.log("Could not parse query as SQL:", e.message);
return;
}
if (sqlParse.verb !== 'SELECT' && sqlParse.verb !== 'DESCRIBE') {
console.log("SQL must be a SELECT or DESCRIBE query");
return;
}
} else {
console.log("no query found please use --query (-q) flag");
return;
}
var expression = sqlParse.expression;
if (verbose) {
console.log('Parsed query as the following plywood expression (as JSON):');
console.log(JSON.stringify(expression, null, 2));
console.log('---------------------------');
}
var dataName = 'data';
var dataSource: string;
if (parsed['data-source']) {
dataSource = parsed['data-source'];
} else if (sqlParse.table) {
dataName = sqlParse.table;
dataSource = sqlParse.table;
} else {
console.log("must have data source");
return;
}
var timeout: number = parsed.hasOwnProperty('timeout') ? parsed['timeout'] : 60000;
var requester: Requester.PlywoodRequester<any>;
requester = druidRequesterFactory({
host: host,
timeout
});
var retry: number = parsed.hasOwnProperty('retry') ? parsed['retry'] : 2;
if (retry > 0) {
requester = helper.retryRequesterFactory({
requester: requester,
retry: retry,
delay: 500,
retryOnTimeout: false
});
}
if (verbose) {
requester = helper.verboseRequesterFactory({
requester: requester
});
}
var concurrent: number = parsed.hasOwnProperty('concurrent') ? parsed['concurrent'] : 2;
if (concurrent > 0) {
requester = helper.concurrentLimitRequesterFactory({
requester: requester,
concurrentLimit: concurrent
});
}
var timeAttribute = '__time';
var filter: Expression = null;
var intervalString: string = parsed['interval'];
if (intervalString) {
try {
var interval = parseIntervalString(intervalString);
} catch (e) {
console.log("Could not parse interval", intervalString);
console.log(e.message);
return;
}
filter = $(timeAttribute).in(interval);
}
var druidContext: Druid.Context = {
timeout
};
if (parsed['skip-cache']) {
druidContext.useCache = false;
druidContext.populateCache = false;
}
var dataset = External.fromJS({
engine: 'druid',
dataSource,
timeAttribute,
allowEternity: allows.indexOf('eternity') !== -1,
allowSelectQueries: allows.indexOf('select') !== -1,
introspectionStrategy: parsed['introspection-strategy'],
filter,
requester,
attributeOverrides,
context: druidContext
});
if (sqlParse.verb === 'DESCRIBE') {
dataset.introspect()
.then((introspectedDataset) => {
console.log(JSON.stringify(introspectedDataset.toJS().attributes, null, 2));
},
(err: Error) => {
console.log(`There was an error getting the metadata: ${err.message}`);
})
.done()
} else if (sqlParse.verb === 'SELECT') {
var context: Datum = {};
context[dataName] = dataset;
expression.compute(context)
.then(
(data: Dataset) => {
var outputStr: string;
switch (output) {
case 'json':
outputStr = JSON.stringify(data, null, 2);
break;
case 'csv':
data = Dataset.fromJS(data.toJS()); // Temp hack
outputStr = data.toCSV();
break;
case 'tsv':
data = Dataset.fromJS(data.toJS()); // Temp hack
outputStr = data.toTSV();
break;
case 'flat':
data = Dataset.fromJS(data.toJS()); // Temp hack
outputStr = JSON.stringify(data.flatten(), null, 2);
break;
default:
outputStr = 'Unknown output type';
break;
}
console.log(outputStr);
},
(err: Error) => {
console.log(`There was an error getting the data: ${err.message}`);
}
).done()
} else {
console.log('Unsupported verb');
}
}
private getFileManagerFor(uri: string): FileManager {
return helper.find(this.fileManagers, (fileManager) => fileManager.uri === uri);
}
private getClusterManagerFor(clusterName: string): ClusterManager {
return helper.find(this.clusterManagers, (clusterManager) => clusterManager.cluster.name === clusterName);
}