chore: Wrote an automatic database creation if not created. Data validation and writing currency rate to the database

.dockerignore

@@ -0,0 +1,7 @@
+Dockerfile
+.gitignore
+.env.sample
+config.sample.hjson
+.git/
+.DS_Store
+.vscode/

.env.sample

@@ -0,0 +1,10 @@
+# Connection secret for postgres. You should change it to a random password
+# Please use only the characters `A-Za-z0-9`, without special characters or spaces
+
+POSTGRES_PASSWORD=my_password
+
+# If you do not know what you are doing, then you should not edit the values below
+###################################################################################
+POSTGRES_DB=kekkai
+DB_HOST=postgres
+POSTGRES_USER=postgres

.gitignore

@@ -8,3 +8,4 @@ config.hjson
 node_modules/
 
 .DS_Store
+postgres-data/

Dockerfile

@@ -0,0 +1,13 @@
+FROM node:20-alpine
+
+WORKDIR /
+
+COPY /package*.json .
+
+RUN npm install
+
+COPY /node_modules /node_modules
+
+COPY . .
+
+CMD ["node", "main.js"]

config.sample.hjson

@@ -3,7 +3,7 @@
         user: DATABASE_USER
         host: DATABASE_HOST
         name: DATABASE_NAME
-        passowrd: DATABASE_PASSWORD
+        password: DATABASE_PASSWORD
     }
     currecy: {
         collecting: {

database/data.js

@@ -0,0 +1,36 @@
+const pg = require('pg');
+const fs = require('fs');
+const hjson = require('hjson');
+
+const config = hjson.parse(fs.readFileSync('config.hjson', 'utf-8'));
+
+const pool = new pg.Pool({
+  user: config['database']['user'],
+  host: config['database']['host'],
+  database: config['database']['name'],
+  password: config['database']['password'],
+  port: 5432
+});
+
+async function create_table() {
+  const schema = fs.readFileSync(
+      '/database/schemas/data.sql',
+      'utf8',
+  );
+
+  const queries = schema
+      .split(';')
+      .map((query) => query.trim())
+      .filter((query) => query);
+
+  for (const query of queries) {
+      try {
+          await pool.query(query);
+      } catch (err) {
+          console.error(err);
+      }
+  }
+}
+
+
+module.exports = { pool, create_table };

database/schemas/data.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS currency(
+    from_currency TEXT NOT NULL,
+    conv_currency TEXT NOT NULL,
+    rate FLOAT NOT NULL,
+    date DATE NOT NULL
+);

docker-compose.yaml

@@ -0,0 +1,29 @@
+services:
+  parser:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - './config.hjson:/config.hjson:ro'
+    depends_on:
+      - postgres
+
+  postgres:
+    image: postgres:latest
+    restart: unless-stopped
+    env_file: .env
+    ports:
+      - '5432:5432'
+    volumes:
+      - ./postgres-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $POSTGRES_USER"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  parser:
+    driver: local
+  postgres:
+    driver: local

main.js

@@ -4,6 +4,8 @@ const hjson = require('hjson');
 const schedule = require('node-schedule');
 const cron = require('cron-validator');
 
+const { validateCurrency } = require('./models/Currency.js');
+const { create_table, pool } = require('./database/data.js');
 const config = hjson.parse(fs.readFileSync('config.hjson', 'utf-8'));
 
 async function main() {
@@ -21,13 +23,12 @@ async function main() {
         const filePath = path.join(servicesDir, file);
         const moduleLoaded = require(filePath);
 
-
+        if (typeof moduleLoaded.parseCurrencies === 'function') 
-        if (typeof moduleLoaded.parseCurrencies === 'function') {
             services.push(moduleLoaded);
-        }
     }
 
     console.log('Loaded parser services:', serviceFiles);
+    await create_table();
 
     schedule.scheduleJob(config['schedule'], async () => {
         console.log('Running scheduled task at:', new Date());
@@ -35,8 +36,24 @@ async function main() {
         for (const srv of services) {
             try {
                 const result = await srv.parseCurrencies();
-                if (result)
+  
-                    console.log(`Result from ${srv.name || 'someService'}:`, result);
+                if (result) {
+                    try {
+                        const currency = await validateCurrency(result);
+
+                        await pool.query(
+                            'INSERT INTO currency (from_currency, conv_currency, rate, date) ' +
+                            'VALUES ($1, $2, $3, $4)', 
+                            [
+                                currency.from_currency,
+                                currency.conv_currency,
+                                currency.rate,
+                                currency.date,
+                            ]);
+                    } catch (validationError) {
+                        console.error(validationError);
+                    }
+                }
             } catch (err) {
                 console.error(`Error in service ${srv.name || 'unknown'}:`, err);
             }

models/Currency.js

@@ -28,7 +28,8 @@ async function validateCurrency(data) {
         const validatedData = await currencySchema.validateAsync(data);
         return validatedData;
     } catch (error) {
-        throw new Error(`Validation error \n${JSON.stringify(error.details[0], null, '\t')}`);
+        const errorMessages = error.details.map(e => e.message).join('; ');
+        throw new Error(`Validation error(s): ${errorMessages}`);
     }
 }
 

package-lock.json

@@ -13,7 +13,8 @@
         "cron-validator": "^1.3.1",
         "hjson": "^3.2.2",
         "joi": "^17.13.3",
-        "node-schedule": "^2.1.1"
+        "node-schedule": "^2.1.1",
+        "pg": "^8.14.1"
       }
     },
     "node_modules/@hapi/hoek": {
@@ -393,6 +394,134 @@
         "node": ">=6"
       }
     },
+    "node_modules/pg": {
+      "version": "8.14.1",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.14.1.tgz",
+      "integrity": "sha512-0TdbqfjwIun9Fm/r89oB7RFQ0bLgduAhiIqIXOsyKoiC/L54DbuAAzIEN/9Op0f1Po9X7iCPXGoa/Ah+2aI8Xw==",
+      "license": "MIT",
+      "dependencies": {
+        "pg-connection-string": "^2.7.0",
+        "pg-pool": "^3.8.0",
+        "pg-protocol": "^1.8.0",
+        "pg-types": "^2.1.0",
+        "pgpass": "1.x"
+      },
+      "engines": {
+        "node": ">= 8.0.0"
+      },
+      "optionalDependencies": {
+        "pg-cloudflare": "^1.1.1"
+      },
+      "peerDependencies": {
+        "pg-native": ">=3.0.1"
+      },
+      "peerDependenciesMeta": {
+        "pg-native": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/pg-cloudflare": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz",
+      "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==",
+      "license": "MIT",
+      "optional": true
+    },
+    "node_modules/pg-connection-string": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.7.0.tgz",
+      "integrity": "sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==",
+      "license": "MIT"
+    },
+    "node_modules/pg-int8": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+      "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/pg-pool": {
+      "version": "3.8.0",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.8.0.tgz",
+      "integrity": "sha512-VBw3jiVm6ZOdLBTIcXLNdSotb6Iy3uOCwDGFAksZCXmi10nyRvnP2v3jl4d+IsLYRyXf6o9hIm/ZtUzlByNUdw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "pg": ">=8.0"
+      }
+    },
+    "node_modules/pg-protocol": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.8.0.tgz",
+      "integrity": "sha512-jvuYlEkL03NRvOoyoRktBK7+qU5kOvlAwvmrH8sr3wbLrOdVWsRxQfz8mMy9sZFsqJ1hEWNfdWKI4SAmoL+j7g==",
+      "license": "MIT"
+    },
+    "node_modules/pg-types": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+      "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+      "license": "MIT",
+      "dependencies": {
+        "pg-int8": "1.0.1",
+        "postgres-array": "~2.0.0",
+        "postgres-bytea": "~1.0.0",
+        "postgres-date": "~1.0.4",
+        "postgres-interval": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/pgpass": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+      "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+      "license": "MIT",
+      "dependencies": {
+        "split2": "^4.1.0"
+      }
+    },
+    "node_modules/postgres-array": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+      "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/postgres-bytea": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
+      "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/postgres-date": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+      "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/postgres-interval": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+      "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+      "license": "MIT",
+      "dependencies": {
+        "xtend": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/proxy-from-env": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
@@ -404,6 +533,24 @@
       "resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz",
       "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==",
       "license": "MIT"
+    },
+    "node_modules/split2": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+      "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+      "license": "ISC",
+      "engines": {
+        "node": ">= 10.x"
+      }
+    },
+    "node_modules/xtend": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.4"
+      }
     }
   }
 }

package.json

@@ -17,6 +17,7 @@
     "cron-validator": "^1.3.1",
     "hjson": "^3.2.2",
     "joi": "^17.13.3",
-    "node-schedule": "^2.1.1"
+    "node-schedule": "^2.1.1",
+    "pg": "^8.14.1"
   }
 }