feat: Add first structure of replicator service

.env

@@ -0,0 +1,7 @@
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5433
+POSTGRES_USER=user
+POSTGRES_PASSWORD=password
+POSTGRES_DB=postgres
+
+DB_URL=postgres://user:password@localhost:5433/postgres?sslmode=disable

.env.example

@@ -0,0 +1,8 @@
+POSTGRES_HOST=localhost // hostname of the PostgreSQL server
+POSTGRES_PORT=5433 // port number of the PostgreSQL server
+POSTGRES_USER=user // username for the PostgreSQL database
+POSTGRES_PASSWORD=password // password for the PostgreSQL database
+POSTGRES_DB=postgres // name of the PostgreSQL database
+
+# Database configuration (2/2) - Remember to set sslmode for production
+DB_URL=postgres://user:password@localhost:5433/postgres?sslmode=disable

.gitignore

@@ -1,8 +1,5 @@
-
-.PHONY: help
-
-
-
-help:
-    @echo "Available commands:"
-    @echo "  make help        - Show this help message"
+.env
+bin
+bin/*
+tmp
+tmp/*

Makefile

@@ -16,11 +16,36 @@ dropBin:
 replicator-service:
 	air -c configs/air/replicator-service.toml
 
+.PHONY: migrate-up
+migrate-up:
+	migrate -path migrations -database $(DB_URL) up
+
+.PHONY: migrate-down
+migrate-down:
+	migrate -path migrations -database $(DB_URL) down
+
+.PHONY: migrate-force
+migrate-force:
+	migrate -path migrations -database $(DB_URL) force $(version)
+
+.PHONY: complete-testing-npm-db
+complete-testing-npm-db:
+	bash scripts/completeNpmDb.sh
+
+.PHONY: create-testing-npm-db
+create-testing-npm-db:
+	bash scripts/createTestingNpmDb.sh
+
 .PHONY: help
 help:
 	@echo "Available commands:"
-	@echo "make help               - Show this help message"
-	@echo "make init               - Initialize pre-push hook"
-	@echo "make build              - Build the project"
-	@echo "make dropBin     	   - Drop the binary files"
-	@echo "make replicator-service - Run the replicator service with air in development mode" 
+	@echo "make help                   	   - Show this help message"
+	@echo "make init                   	   - Initialize pre-push hook"
+	@echo "make build                  	   - Build the project"
+	@echo "make dropBin                    - Drop the binary files"
+	@echo "make replicator-service         - Run the replicator service with air in development mode" 
+	@echo "make migrate-up                 - Run database migrations up"
+	@echo "make migrate-down               - Run database migrations down"
+	@echo "make migrate-force              - Force database migration to a specific version (usage: make migrate-force version=<version_number>)"
+	@echo "make complete-testing-npm-db    - Complete testing npm database"
+	@echo "make create-testing-npm-db      - Create testing npm database"

README.md

@@ -8,8 +8,8 @@ API for tracking vulnerability occurrence frequencies.
 
 ```bash
 make init
-2. Install Go
 ```
+2. Install Go
 
 This step depends on your operating system.
 
@@ -51,3 +51,17 @@ go install github.com/air-verse/air@latest
 ```bash
 make replicator-service
 ```
+
+4. pick up Postgres db
+
+- podman
+
+```bash
+podman compose up -d
+```
+
+ - or docker
+
+```bash
+docker compose up -d
+```

cmd/replicator-service/main.go

@@ -1,5 +1,17 @@
 package main
 
+import (
+	"fmt"
+
+	"github.com/trustpkg/trustpkg-api/db"
+	"github.com/trustpkg/trustpkg-api/internal/npm"
+)
+
 func main() {
-	println("Hello, World!")
-}
+	db.ConnectDb()
+
+	err := npm.Pipeline()
+	if err != nil {
+		fmt.Println("commit error: ", err)
+	}
+}

db/postgres.go

@@ -0,0 +1,65 @@
+package db
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	goDotEnv "github.com/joho/godotenv"
+)
+
+type postgresConfig struct {
+	host     string
+	port     string
+	user     string
+	password string
+	dbName   string
+}
+
+var Pool *pgxpool.Pool
+
+func ConnectDb() {
+	if err := goDotEnv.Load(".env"); err != nil {
+		log.Println(".env file not found, using system environment variables")
+	}
+
+	dsn := strings.TrimSpace(os.Getenv("DB_URL"))
+	if dsn == "" {
+		var dbConfig = postgresConfig{
+			host:     os.Getenv("POSTGRES_HOST"),
+			port:     os.Getenv("POSTGRES_PORT"),
+			user:     os.Getenv("POSTGRES_USER"),
+			password: os.Getenv("POSTGRES_PASSWORD"),
+			dbName:   os.Getenv("POSTGRES_DB"),
+		}
+
+		if dbConfig.host == "" || dbConfig.port == "" || dbConfig.user == "" || dbConfig.dbName == "" {
+			fmt.Fprintln(os.Stderr, "Missing DB config. Set DB_URL or POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB (copy .env.example to .env).")
+			os.Exit(1)
+		}
+
+		dsn = fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=disable", dbConfig.user, dbConfig.password, dbConfig.host, dbConfig.port, dbConfig.dbName)
+	}
+
+	pool, err := pgxpool.New(context.Background(), dsn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err)
+		os.Exit(1)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	err = pool.Ping(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to ping database: %v\n", err)
+		os.Exit(1)
+	}
+
+	Pool = pool
+
+	log.Println("Connected to postgres database successfully")
+}

docker-compose.couchDb.yml

@@ -0,0 +1,11 @@
+services:
+  couchdb:
+    image: docker.io/library/couchdb:latest
+    restart: always
+    ports:
+      - 3200:5984
+    environment:
+      COUCHDB_USER: admin
+      COUCHDB_PASSWORD: password
+      TZ: UTC
+

docker-compose.yml

@@ -21,7 +21,7 @@ services:
       - 8081:8080
     depends_on: 
       - db
-    network:
+    networks:
       - trustpkg-net
 
 volumes:

go.mod

@@ -1,3 +1,28 @@
-module github.com/TymekGluch/trustpkg-api
+module github.com/trustpkg/trustpkg-api
 
 go 1.26.5
+
+require (
+	github.com/jackc/pgx/v5 v5.10.0
+	github.com/joho/godotenv v1.5.1
+)
+
+require (
+	github.com/ebitengine/purego v0.10.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	golang.org/x/sys v0.41.0 // indirect
+)
+
+require (
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/shirou/gopsutil/v4 v4.26.7
+	golang.org/x/sync v0.17.0 // indirect
+	golang.org/x/text v0.29.0 // indirect
+)

go.sum

@@ -0,0 +1,50 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE=
+github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
+github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc=
+github.com/shirou/gopsutil/v4 v4.26.7/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
+github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
+github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
+golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/adaptive-worker/constants.go

@@ -0,0 +1,25 @@
+package adaptiveWorker
+
+const (
+	minWorkers       = 2
+	maxWorkersPerCpu = 16
+
+	maxWorkerHeadroom = 0.2
+
+	maxCpuUsage    = 0.6
+	maxMemoryUsage = 0.8
+
+	timeFromRequestWorker = 30
+
+	skipCount = 2
+
+	badUsagePoints       = 0
+	RegularUsagePoints   = 1
+	GoodUsagePoints      = 2
+	ExcellentUsagePoints = 3
+
+	usageStateBad       string = "bad"
+	usageStateRegular   string = "regular"
+	usageStateGood      string = "good"
+	usageStateExcellent string = "excellent"
+)

internal/adaptive-worker/controler.go

@@ -0,0 +1,69 @@
+package adaptiveWorker
+
+type Controler struct {
+	simultaneousWorkers int
+}
+
+func (controler *Controler) Adjust(count int) {
+	controler.simultaneousWorkers = count
+}
+
+func (controler *Controler) CalculateWorkers() error {
+	resourceUsage, err := CheckResourceUsage()
+	if err != nil {
+		controler.Adjust(minWorkers)
+
+		return err
+	}
+
+	maxSimultaneousWorkers := getMaxWorkers(resourceUsage.cpuCount)
+	maxSimultaneousWorkersHeadroom := getMaxWorkersHeadroom(resourceUsage.cpuCount)
+
+	calculedData := calculedResourcesUsage{
+		loadPercent: resourceUsage.loadPercent,
+		cpuPercent:  resourceUsage.cpuPercent,
+		ramPercent:  resourceUsage.ramPercent,
+	}
+
+	isAtBoundary := controler.simultaneousWorkers >= maxSimultaneousWorkers
+	canEnterHeadroom := isAtBoundary && controler.simultaneousWorkers+skipCount <= maxSimultaneousWorkersHeadroom
+
+	var simultaneousWorkers int
+	if controler.simultaneousWorkers == 0 {
+		simultaneousWorkers = minWorkers
+	} else {
+		simultaneousWorkers = controler.simultaneousWorkers
+	}
+
+	if getResourceUsageState(calculedData) == usageStateExcellent {
+		if isAtBoundary && canEnterHeadroom {
+			controler.Adjust(simultaneousWorkers + skipCount)
+		} else {
+			controler.Adjust(simultaneousWorkers)
+		}
+
+		return nil
+	}
+
+	if getResourceUsageState(calculedData) == usageStateGood {
+		controler.Adjust(simultaneousWorkers + skipCount)
+
+		return nil
+	}
+
+	if getResourceUsageState(calculedData) == usageStateRegular {
+		controler.Adjust(simultaneousWorkers)
+
+		return nil
+	}
+
+	if getResourceUsageState(calculedData) == usageStateBad {
+		controler.Adjust(simultaneousWorkers - skipCount)
+
+		return nil
+	}
+
+	controler.Adjust(minWorkers)
+
+	return nil
+}