Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Storage Service #1687

Open
wants to merge 48 commits into
base: b4.1.0
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
792cb3c
Merge pull request #1 from MarceloPilatti/b4.1.0
rosamariakato Apr 12, 2019
706dd5c
Save Storager with GUI interface
raphaelrpl Apr 15, 2019
5eefc0a
fix crud operation of storage
raphaelrpl Apr 16, 2019
3b22cdc
updates
rosamariakato Apr 16, 2019
8f93f0c
testando queues
rosamariakato Apr 17, 2019
51bd585
Queue and cron events
rosamariakato Apr 18, 2019
1b200d1
Messages updating
rosamariakato Apr 22, 2019
51ec63a
Messages control
rosamariakato Apr 23, 2019
aaa281e
storages
rosamariakato Apr 24, 2019
ac90c52
organizing the functions
rosamariakato Apr 25, 2019
ac65b0a
Adjusts
rosamariakato Apr 26, 2019
20dd086
Update storager component
raphaelrpl Apr 29, 2019
4e99d27
Merge remote-tracking branch 'rosarepo/b_storage' into b_storage
raphaelrpl Apr 29, 2019
d4c3ba9
Merge pull request #2 from raphaelrpl/b_storage
rosamariakato Apr 30, 2019
48271cb
setting Log
rosamariakato Apr 30, 2019
de57904
setting Log
rosamariakato Apr 30, 2019
94b478a
fix update storager. TODO: handle moment time error
raphaelrpl May 2, 2019
ecbaa69
Logs
rosamariakato May 2, 2019
56053d4
Fix storage update and GUI minor fixes
raphaelrpl May 3, 2019
0b7d5f5
logs and adjusts front-end
rosamariakato May 3, 2019
b738d96
Fix storager crud on GUI
raphaelrpl May 6, 2019
de58aa4
merge fix
raphaelrpl May 6, 2019
1493dc0
Merge pull request #3 from raphaelrpl/b_storage
rosamariakato May 7, 2019
0a9c19c
Set storage validation of data series to inactive
raphaelrpl May 7, 2019
e730dd1
table messages update
rosamariakato May 7, 2019
ad493b5
Organization of directories and files
rosamariakato May 8, 2019
0b63e52
Update storage in the service handling
raphaelrpl May 9, 2019
759983e
Merge remote-tracking branch 'rosarepo/b_storage' into b_storage
raphaelrpl May 10, 2019
9e83dc1
merge fix
raphaelrpl May 10, 2019
bde1852
Merge pull request #4 from raphaelrpl/b_storage
rosamariakato May 10, 2019
9b5b40e
Storager to storage
rosamariakato May 10, 2019
564a9b0
Front-end adjusts, cron, logger
rosamariakato May 10, 2019
3c8ac48
storage adjusts
rosamariakato May 13, 2019
c1a63e0
Edit storage from status
rosamariakato May 13, 2019
6e195d3
translate to portuguese
rosamariakato May 14, 2019
524853a
Time unit
rosamariakato May 14, 2019
f472466
Keep data controls
rosamariakato May 15, 2019
30b43e8
Storages translations
rosamariakato May 15, 2019
b7bc327
Documentation
rosamariakato May 15, 2019
9a7a08e
merge with b4.1.0
rosamariakato May 15, 2019
e89e43c
restore code
rosamariakato May 15, 2019
2d17c31
merge
rosamariakato May 15, 2019
95706e6
update Storage and analisys
rosamariakato May 15, 2019
e441f3e
pg version
rosamariakato May 16, 2019
d446533
nessages control
rosamariakato May 16, 2019
d316efb
Start message
rosamariakato May 16, 2019
670fdbc
more translations
rosamariakato May 17, 2019
6f44a76
table name and duplicated key
rosamariakato May 17, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions webapp/controllers/administration/Storage.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
'use strict';

// dependencies
const Enums = require('./../../core/Enums');
const StorageFacade = require('./../../core/facade/storage');
const makeTokenParameters = require('../../core/Utils').makeTokenParameters;

/**
* It exports a object with Storage controllers (get/new/edit)
* @return {Object} A object with controllers with http method as key (get/new/edit)
*/
module.exports = function(app) {
return {
get: function(request, response) {
const parameters = makeTokenParameters(request.query.token, app);
const hasProjectPermission = request.session.activeProject.hasProjectPermission;
parameters.hasProjectPermission = hasProjectPermission;
response.render('administration/storages', Object.assign({}, parameters, {"Enums": Enums}));
},

new: function(request, response) {
response.render('administration/storage');
},

edit: async (request, response) => {
const storageId = request.params.id;
const hasProjectPermission = request.session.activeProject.hasProjectPermission;

const facade = new StorageFacade();

try {
await facade.get(storageId);

response.render('administration/storage', {hasProjectPermission: hasProjectPermission, storage: storageId });
} catch (err) {
response.render('base/404');
}
}
};
};
27 changes: 16 additions & 11 deletions webapp/controllers/api/Service.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,25 @@ module.exports = function(app) {
return {
get: function(request, response) {
var type = request.query.type;
var serviceId = request.query.serviceId;


if (!serviceId) {
var serviceId = request.query.serviceId;
//todo: improve it
var restriction = {};
switch (type) {
case "COLLECT":
restriction = {service_type_id: 1};
break;
case "ANALYSIS":
restriction = {service_type_id: 2};
break;
default:
break;

if (isNaN(type)) {
switch (type) {
case "COLLECT":
restriction = {service_type_id: 1};
break;
case "ANALYSIS":
restriction = {service_type_id: 2};
break;
default:
break;
}
} else {
restriction.service_type_id = type;
}

return DataManager.listServiceInstances(restriction).then(function(services) {
Expand Down
68 changes: 68 additions & 0 deletions webapp/controllers/api/storage.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// dependencies
const storageFacade = require('../../core/facade/storage');

const mergeStorageWithProject = (request, storage) => (
{ project_id: request.session.activeProject.id, ...storage }
);

/**
* It exports a object with Storage controllers (get/new/edit)
* @return {Object} A object with controllers with http method as key (get/new/edit)
*/
module.exports = function(app) {
return {
get: async (request, response) => {
try {
let output = null;

const facade = new storageFacade();

if (request.params.id) {
output = await facade.get(request.params.id);
} else {
output = await facade.list();
}

response.json(output);
} catch (err) {
response.status(400);
response.json({ error: err.message });
}
},
save: async (request, response) => {
try {
const storage = await new storageFacade().save(mergeStorageWithProject(request, request.body));
response.json(storage);
} catch (err) {
response.status(err.code);
response.json(err.getErrors());
}
},
put: async (request, response) => {
try {
const facade = new storageFacade();

await facade.update(request.params.id, mergeStorageWithProject(request, request.body));

response.status(204);
response.json({});
} catch (err) {
response.status(err.code);
response.json(err.getErrors());
}
},
delete: async (request, response) => {
try {
const facade = new storageFacade();

await facade.delete(request.params.id);

response.status(204)
response.json({});
} catch (err) {
response.status(err.code);
response.json(err.getErrors());
}
}
};
};
8 changes: 7 additions & 1 deletion webapp/controllers/configuration/Status.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,11 @@ module.exports = function(app) {
DataManager.listViews(),
DataManager.listAlerts(),
DataManager.listInterpolators(),
DataManager.listStorages(),
DataManager.listProjects()
])

.spread(function(collectors, analysisList, views, alerts, interpolators, projects) {
.spread(function(collectors, analysisList, views, alerts, interpolators, storages, projects) {
var outputAnalysis = [];
analysisList.forEach(function(analysis) {
outputAnalysis.push(analysis.rawObject());
Expand All @@ -41,13 +42,18 @@ module.exports = function(app) {
return interpolator.rawObject();
});

var outputStorages = storages.map(function(storage){
return storage;
});

var renderParams = {
"Enums": Enums,
"analysis": outputAnalysis,
"collectors": outputCollectors,
"views": outputViews,
"alerts": outputAlerts,
"interpolators": outputInterpolators,
"storages": outputStorages,
"projects": projects,
"parameters": null
};
Expand Down
28 changes: 28 additions & 0 deletions webapp/controllers/configuration/Storages.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
"use strict";

var DataManager = require("./../../core/DataManager");
var ScheduleType = require("./../../core/Enums").ScheduleType;
var makeTokenParameters = require('../../core/Utils').makeTokenParameters;

module.exports = function(app) {
return {
get: function(request, response) {
var parameters = makeTokenParameters(request.query.token, app);
var hasProjectPermission = request.session.activeProject.hasProjectPermission;
parameters.hasProjectPermission = hasProjectPermission;
response.render("configuration/storages", parameters);
},
new: function(request, response) {
return response.render("configuration/storages", {ScheduleType: ScheduleType});
},
edit: function(request, response) {
var hasProjectPermission = request.session.activeProject.hasProjectPermission;
DataManager.getStorage({id: parseInt(request.params.id)})
.then(function(storage) {
return response.render("configuration/storages", {storage: storage.rawObject(), ScheduleType: ScheduleType, hasProjectPermission: hasProjectPermission});
}).catch(function(err) {
return response.render("base/404");
});
}
}
};
1 change: 1 addition & 0 deletions webapp/core/Application.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
var fs = require("fs");
var path = require("path");
var bcrypt = require('bcrypt');
var util = require('util');

/**
* It defines a cache object (private) with TerraMA² settings
Expand Down
75 changes: 73 additions & 2 deletions webapp/core/DataManager.js
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ var DataManager = module.exports = {
init: function(dbConfigPath, callback) {
var self = this;
logger.info("Initializing database...");
logger.info(dbConfigPath);

return Database.init(dbConfigPath).then(function(dbORM) {
logger.info("Database loaded.");
Expand Down Expand Up @@ -139,6 +140,7 @@ var DataManager = module.exports = {
inserts.push(models.db.ServiceType.create({id: Enums.ServiceType.VIEW, name: "VIEW"}));
inserts.push(models.db.ServiceType.create({id: Enums.ServiceType.ALERT, name: "ALERT"}));
inserts.push(models.db.ServiceType.create({id: Enums.ServiceType.INTERPOLATION, name: "INTERPOLATION"}));
inserts.push(models.db.ServiceType.create({id: Enums.ServiceType.STORAGE, name: "STORAGE"}));

// data provider type defaults
inserts.push(self.addDataProviderType({id: 1, name: "FILE", description: "Desc File"}));
Expand Down Expand Up @@ -214,11 +216,19 @@ var DataManager = module.exports = {
interpolationService.port = 6547;
interpolationService.service_type_id = Enums.ServiceType.INTERPOLATION;

var storageService = Object.assign({}, collectorService);
storageService.name = "Local Storage";
storageService.description = "Local service for Storage";
storageService.pathToBinary = "../storage/storage_service.js",
storageService.port = 5000;
storageService.service_type_id = Enums.ServiceType.STORAGE;

servicesInsert.push(self.addServiceInstance(collectorService));
servicesInsert.push(self.addServiceInstance(analysisService));
servicesInsert.push(self.addServiceInstance(viewService));
servicesInsert.push(self.addServiceInstance(alertService));
servicesInsert.push(self.addServiceInstance(interpolationService));
servicesInsert.push(self.addServiceInstance(storageService));
}
return Promise.all(servicesInsert);
});
Expand Down Expand Up @@ -3231,7 +3241,7 @@ var DataManager = module.exports = {
return reject(new exceptions.CollectorError("Could not find collector. " + err.toString()));
});
} else {
logger.error("Retrieved null while getting collector", collectorResult);
logger.error("Retrieved null while getting collector", restriction, collectorResult);
return reject(new exceptions.CollectorErrorNotFound("Could not find collector. "));
}
}).catch(function(err) {
Expand Down Expand Up @@ -6629,5 +6639,66 @@ var DataManager = module.exports = {
return reject(err);
});
});
}
},

/**
* It retrieves all storages from database
*
* @param {Object} restriction - A query restriction
* @param {Object} options - A query options
* @param {Transaction} options.transaction - An ORM transaction
* @returns {Promise<Array<Object>>}
*/
listStorages: function(restriction, options) {
var self = this;
return new Promise(function(resolve, reject) {
var _reject = function(err) {
logger.error(err);
reject(err);
};

return models.db.Storages.findAll({})
.then(function(storages){
var output = [];
storages.forEach(function(storage) {
output.push(storage.get());
});
resolve(output);
}).catch(function(err) {
reject(new Error("Could not retrieve storage " + err.toString()));
});

});
},

/**
* It retrieves a storage of database from given restriction
*
* @param {Object} restriction - A query restriction
* @param {Object} options - A query options
* @param {Transaction} options.transaction - An ORM transaction
* @returns {Promise<Storage>}
*/
getStorage: function(restriction, options) {
var self = this;
return new Promise(function(resolve, reject) {
self.listStorages(restriction, options)
.then(function(storages) {
if (storages.length === 0) {
return reject(new Error("No storage retrieved"));
}

if (storages.length > 1) {
return reject(new Error("Get operation retrieved more than a storage"));
}

return resolve(storages[0]);
})
.catch(function(err) {
return reject(err);
});
});
},

};

7 changes: 6 additions & 1 deletion webapp/core/Enums.js
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,12 @@ module.exports = {
* Defines Interpolation Service
* @type {number}
*/
INTERPOLATION: 5
INTERPOLATION: 5,
/**
* Defines Storage Service
* @type {number}
*/
STORAGE: 6
},

DataSeriesType: {
Expand Down
17 changes: 17 additions & 0 deletions webapp/core/Exceptions.js
Original file line number Diff line number Diff line change
Expand Up @@ -406,3 +406,20 @@ errors.InterpolatorError = function(message) {
this.name = 'InterpolatorError';
};
util.inherits(errors.InterpolatorError, errors.BaseError);

class TcpError extends Error {
constructor(service, message, process = null) {
super(message);

this.service = service;
this.process = process;
}
}
class NoSuchConnectionError extends TcpError {
constructor(service, process = null) {
super(service, `There is no active connection to the service ${service.name || service.id || service}`, process);
}
}

errors.TcpError = TcpError;
errors.NoSuchConnectionError = NoSuchConnectionError;
1 change: 1 addition & 0 deletions webapp/core/Logger.js
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
* @returns {string} Formatted string to log
*/
function loggerFormat(args) {
format = new Date().toLocaleTimeString();
return NodeFormat("[%s] %s %s", format, winston.config.colorize(args.level), args.message);
}

Expand Down
15 changes: 15 additions & 0 deletions webapp/core/ObjectDependencies.js
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,21 @@ var ObjectDependencies = function(json){
);
}
}
else if (json.objectType == "Storage"){
for(var i = 0, idsLength = json.ids.length; i < idsLength; i++) {
promises.push(
DataManager.getStorage({id: json.ids[i]}).then(function(storage){
if(output["Storages_" + storage.id] === undefined) output["Storages_" + storage.id] = {};

if(output["Storages_" + storage.id] === undefined) output["Storages_" + storage.id] = {};
var storageDataseriesListPromises = [getDataSeriesDependencies(storage.data_series_input, null, "Storages_" + storage.id)];
storageDataseriesListPromises.push(getDataSeriesDependencies(storage.data_series_output, null, "Storages_" + storage.id));

return Promise.all(storageDataseriesListPromises).catch(_emitError);
})
);
}
}

return Promise.all(promises).then(function() {
return Promise.resolve({ status: 200, data: output, projectId: json.projectId });
Expand Down
Loading