start migration to drizzle and postgres

docker-compose.yml

@@ -1,18 +1,12 @@
-version: "3.9"
-
-volumes:
-  mongo-data:
-
 services:
   db:
-    image: mongo:5
-    container_name: osem-dev-mongo
+    image: timescale/timescaledb-ha:pg15-latest
+    command:
+      - -cshared_preload_libraries=timescaledb,pg_cron
+    restart: always
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=opensensemap
     ports:
-      - "27017:27017"
-    volumes:
-      - mongo-data:/data/db
-      # - ./dumps/boxes:/exports/boxes
-      # - ./dumps/measurements:/exports/measurements
-      - ./.scripts/mongodb/osem_admin.sh:/docker-entrypoint-initdb.d/osem_admin.sh
-      # - ./.scripts/mongodb/osem_seed_boxes.sh:/docker-entrypoint-initdb.d/osem_seed_boxes.sh
-      # - ./.scripts/mongodb/osem_seed_measurements.sh:/docker-entrypoint-initdb.d/osem_seed_measurements.sh
+      - 5432:5432

packages/api/app.js

@@ -53,21 +53,26 @@ if (config.get('logLevel') === 'debug') {
   server.use(debugLogger);
 }
 
-db.connect()
-  .then(function () {
-    // attach Routes
+const run = async function () {
+  try {
+    // TODO: Get a client from the Pool and test connection
+    await db.connect();
+
     routes(server);
 
     // start the server
     server.listen(Number(config.get('port')), function () {
       stdLogger.logger.info(`${server.name} listening at ${server.url}`);
       postToMattermost(`openSenseMap API started. Version: ${getVersion}`);
     });
-  })
-  .catch(function (err) {
-    stdLogger.logger.fatal(err, 'Couldn\'t connect to MongoDB. Exiting...');
+  } catch (error) {
+    stdLogger.logger.fatal(error, 'Couldn\'t connect to PostgreSQL. Exiting...');
     process.exit(1);
-  });
+  }
+};
+
+// 🔥 Fire up API
+run();
 
 // InternalServerError is the only error we want to report to Honeybadger..
 server.on('InternalServer', function (req, res, err, callback) {

packages/api/lib/controllers/usersController.js

@@ -15,6 +15,7 @@ const { User } = require('@sensebox/opensensemap-api-models'),
     refreshJwt,
     invalidateToken,
   } = require('../helpers/jwtHelpers');
+const { createUser, findUserByNameOrEmail, checkPassword } = require('@sensebox/opensensemap-api-models/src/user/user');
 
 /**
  * define for nested user parameter for box creation request
@@ -53,10 +54,11 @@ const { User } = require('@sensebox/opensensemap-api-models'),
  * @apiSuccess (Created 201) {Object} data `{ "user": {"name":"fullname","email":"[email protected]","role":"user","language":"en_US","boxes":[],"emailIsConfirmed":false} }`
  */
 const registerUser = async function registerUser (req, res) {
-  const { email, password, language, name, integrations } = req._userParams;
+  const { email, password, language, name } = req._userParams;
 
   try {
-    const newUser = await new User({ name, email, password, language, integrations }).save();
+    const newUser = await createUser(name, email, password, language);
+
     postToMattermost(
       `New User: ${newUser.name} (${redactEmail(newUser.email)})`
     );
@@ -101,18 +103,15 @@ const signIn = async function signIn (req, res) {
   const { email: emailOrName, password } = req._userParams;
 
   try {
-    // lowercase for email
-    const user = await User.findOne({
-      $or: [{ email: emailOrName.toLowerCase() }, { name: emailOrName }],
-    }).exec();
+    const user = await findUserByNameOrEmail(emailOrName);
 
     if (!user) {
       return Promise.reject(
         new ForbiddenError('User and or password not valid!')
       );
     }
 
-    if (await user.checkPassword(password)) {
+    if (await checkPassword(password, user.password)) {
       const { token, refreshToken } = await createToken(user);
 
       res.send(200, {

packages/api/lib/helpers/jwtHelpers.js

@@ -39,14 +39,18 @@ const createToken = function createToken (user) {
       // it is a HMAC of the jwt string
       const refreshToken = hashJWT(token);
       try {
-        await user.update({
-          $set: {
-            refreshToken,
-            refreshTokenExpires: moment.utc()
-              .add(Number(refresh_token_validity_ms), 'ms')
-              .toDate()
-          }
-        }).exec();
+        // TODO: do we need a new table for tokens???
+        user.refreshToken = refreshToken;
+        user.refreshTokenExpires = moment.utc().add(Number(refresh_token_validity_ms), 'ms')
+          .toDate();
+        // await user.update({
+        //   $set: {
+        //     refreshToken,
+        //     refreshTokenExpires: moment.utc()
+        //       .add(Number(refresh_token_validity_ms), 'ms')
+        //       .toDate()
+        //   }
+        // }).exec();
 
         return resolve({ token, refreshToken });
       } catch (err) {

packages/models/drizzle.config.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const config = require('config');
+
+config.util.setModuleDefaults('opensensemap-migrations', {
+  db: {
+    host: 'localhost',
+    port: 5432,
+    user: 'postgres',
+    userpass: 'postgres',
+    db: 'opensensemap',
+    database_url: ''
+  },
+});
+
+const getDBUri = function getDBUri () {
+  // get uri from config
+  const uri = config.get('opensensemap-migrations.db.database_url');
+  if (uri) {
+    return uri;
+  }
+
+  // otherwise build uri from config supplied values
+  const { user, userpass, host, port, db } = config.get('opensensemap-migrations.db');
+
+  return `postgresql://${user}:${userpass}@${host}:${port}/${db}`;
+};
+
+/** @type { import("drizzle-kit").Config } */
+module.exports = {
+  schema: './schema/*',
+  out: './migrations',
+  dialect: 'postgresql',
+  dbCredentials: {
+    url: getDBUri()
+  }
+};

packages/models/index.js

@@ -9,12 +9,11 @@ const config = require('config');
 config.util.setModuleDefaults('openSenseMap-API-models', {
   db: {
     host: 'localhost',
-    port: 27017,
-    user: 'admin',
-    userpass: 'admin',
-    authsource: 'OSeM-api',
-    db: 'OSeM-api',
-    mongo_uri: '',
+    port: 5432,
+    user: 'postgres',
+    userpass: 'postgres',
+    db: 'opensensemap',
+    database_url: ''
   },
   integrations: {
     ca_cert: '',
@@ -25,29 +24,29 @@ config.util.setModuleDefaults('openSenseMap-API-models', {
       port: 6379,
       username: '',
       password: '',
-      db: 0,
+      db: 0
     },
     mailer: {
       url: '',
       origin: '',
-      queue: 'mails',
+      queue: 'mails'
     },
     mqtt: {
-      url: '',
-    },
+      url: ''
+    }
   },
   password: {
     min_length: 8,
-    salt_factor: 13,
+    salt_factor: 13
   },
   claims_ttl: {
     amount: 1,
-    unit: 'd',
+    unit: 'd'
   },
   pagination: {
     max_boxes: 3
   },
-  image_folder: './userimages/',
+  image_folder: './userimages/'
 });
 
 const { model: Box } = require('./src/box/box'),

packages/models/package.json

@@ -7,26 +7,36 @@
   "dependencies": {
     "@grpc/grpc-js": "^1.9.4",
     "@grpc/proto-loader": "^0.7.10",
+    "@paralleldrive/cuid2": "^2.2.2",
     "@sensebox/osem-protos": "^1.1.0",
     "@sensebox/sketch-templater": "1.13.1",
     "bcrypt": "^5.1.1",
     "bullmq": "^4.12.3",
     "config": "^3.3.6",
+    "drizzle-orm": "^0.33.0",
     "got": "^11.8.2",
     "isemail": "^3.0.0",
     "jsonpath": "^1.1.1",
     "lodash.isequal": "^4.5.0",
     "moment": "^2.29.4",
     "mongoose": "^5.13.20",
     "mongoose-timestamp": "^0.6",
+    "pg": "^8.13.0",
     "pino": "^8.8.0",
     "uuid": "^8.3.2"
   },
   "scripts": {
+    "db:generate": "drizzle-kit generate",
+    "db:generate-custom": "drizzle-kit generate --custom",
+    "db:migrate": "drizzle-kit migrate",
+    "db:studio": "drizzle-kit studio",
     "version": "node .scripts/npm_version-update_changelog.js && git add CHANGELOG.md",
     "test": "mocha test/waitForDatabase test/index.js"
   },
   "publishConfig": {
     "access": "public"
+  },
+  "devDependencies": {
+    "drizzle-kit": "^0.24.2"
   }
 }

packages/models/schema/enum.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const { pgEnum } = require('drizzle-orm/pg-core');
+
+const deviceModel = pgEnum('model', [
+  'HOME_V2_LORA'
+]);
+
+const exposure = pgEnum('exposure', [
+  'indoor',
+  'outdoor',
+  'mobile',
+  'unknown'
+]);
+
+const status = pgEnum('status', [
+  'active',
+  'inactive',
+  'old'
+]);
+
+module.exports = {
+  deviceModel,
+  exposure,
+  status
+};

packages/models/schema/measurement.js

@@ -0,0 +1,75 @@
+'use strict';
+
+const { pgTable, text, timestamp, doublePrecision, unique, pgMaterializedView, integer } = require('drizzle-orm/pg-core');
+
+/**
+ * Table definition
+ */
+const measurement = pgTable('measurement', {
+  sensorId: text('sensor_id').notNull(),
+  time: timestamp('time', { precision: 3, withTimezone: true }).defaultNow()
+    .notNull(),
+  value: doublePrecision('value')
+}, (t) => ({
+  unq: unique().on(t.sensorId, t.time)
+}));
+
+/**
+ * Views
+ */
+const measurement10minView = pgMaterializedView('measurement_10min', {
+  sensorId: text('sensor_id'),
+  time: timestamp('time', { precision: 3, withTimezone: true }),
+  value: doublePrecision('avg_value'),
+  total_values: integer('total_values'),
+  min_value: doublePrecision('min_value'),
+  max_value: doublePrecision('max_value')
+}).existing();
+
+const measurements1hourView = pgMaterializedView('measurement_1hour', {
+  sensorId: text('sensor_id'),
+  time: timestamp('time', { precision: 3, withTimezone: true }),
+  value: doublePrecision('avg_value'),
+  total_values: integer('total_values'),
+  min_value: doublePrecision('min_value'),
+  max_value: doublePrecision('max_value')
+}).existing();
+
+const measurements1dayView = pgMaterializedView('measurement_1day', {
+  sensorId: text('sensor_id'),
+  time: timestamp('time', { precision: 3, withTimezone: true }),
+  value: doublePrecision('avg_value'),
+  total_values: integer('total_values'),
+  min_value: doublePrecision('min_value'),
+  max_value: doublePrecision('max_value')
+}).existing();
+
+const measurements1monthView = pgMaterializedView('measurement_1month', {
+  sensorId: text('sensor_id'),
+  time: timestamp('time', { precision: 3, withTimezone: true }),
+  value: doublePrecision('avg_value'),
+  total_values: integer('total_values'),
+  min_value: doublePrecision('min_value'),
+  max_value: doublePrecision('max_value')
+}).existing();
+
+const measurements1yearView = pgMaterializedView('measurement_1year', {
+  sensorId: text('sensor_id'),
+  time: timestamp('time', { precision: 3, withTimezone: true }),
+  value: doublePrecision('avg_value'),
+  total_values: integer('total_values'),
+  min_value: doublePrecision('min_value'),
+  max_value: doublePrecision('max_value')
+}).existing();
+
+
+module.exports = {
+  table: measurement,
+  views: {
+    measurement10minView,
+    measurements1hourView,
+    measurements1dayView,
+    measurements1monthView,
+    measurements1yearView
+  }
+};