feat: create oracle connector (#310)

gateway/package.json

@@ -45,6 +45,7 @@
     "@azimutt/connector-mariadb": "^0.1.1",
     "@azimutt/connector-mongodb": "^0.1.1",
     "@azimutt/connector-mysql": "^0.1.2",
+    "@azimutt/connector-oracle": "workspace:^",
     "@azimutt/connector-postgres": "^0.1.6",
     "@azimutt/connector-snowflake": "^0.1.1",
     "@azimutt/connector-sqlserver": "^0.1.1",

gateway/src/services/connector.ts

@@ -7,6 +7,7 @@ import {mysql} from "@azimutt/connector-mysql"
 import {postgres} from "@azimutt/connector-postgres"
 import {snowflake} from "@azimutt/connector-snowflake"
 import {sqlserver} from "@azimutt/connector-sqlserver"
+import {oracle} from "@azimutt/connector-oracle"
 
 const connectors: Record<DatabaseKind, Connector | undefined> = {
     bigquery: bigquery,
@@ -17,7 +18,7 @@ const connectors: Record<DatabaseKind, Connector | undefined> = {
     mariadb: mariadb,
     mongodb: mongodb,
     mysql: mysql,
-    oracle: undefined,
+    oracle: oracle,
     postgres: postgres,
     redis: undefined,
     snowflake: snowflake,

libs/connector-oracle/.gitignore

@@ -0,0 +1,5 @@
+node_modules
+out
+local
+src/**/*.js
+*.tgz

libs/connector-oracle/.npmignore

@@ -0,0 +1,7 @@
+local
+resources
+src/*.js
+src/*.test.ts
+jest.config.js
+tsconfig.json
+*.tgz

libs/connector-oracle/README.md

@@ -0,0 +1,75 @@
+# Oracle connector
+
+This library allows to connect to [Oracle](https://www.oracle.com/database), extract its schema and more...
+
+It lists all schemas, tables, columns, relations and types and format them in a JSON Schema.
+
+This library is made by [Azimutt](https://azimutt.app) to allow people to explore their Oracle database.
+It's accessible through the [Desktop app](../../desktop) (soon), the [CLI](https://www.npmjs.com/package/azimutt) or even the website using the [gateway](../../gateway) server.
+
+**Feel free to use it and even submit PR to improve it:**
+
+## Publish
+
+- update `package.json` version
+- update lib versions (`pnpm -w run update` + manual)
+- test with `pnpm run dry-publish` and check `azimutt-connector-oracle-x.y.z.tgz` content
+- launch `pnpm publish --access public`
+
+View it on [npm](https://www.npmjs.com/package/@azimutt/connector-oracle).
+
+## Dev
+
+If you need to develop on multiple libs at the same time (ex: want to update a connector and try it through the CLI), depend on local libs but publish & revert before commit.
+
+- Depend on a local lib: `pnpm add <lib>`, ex: `pnpm add @azimutt/models`
+- "Publish" lib locally by building it: `pnpm run build`
+
+## Oracle Setup
+
+### Run in Docker
+
+You can use the free version of Oracle Database
+
+```bash
+docker pull container-registry.oracle.com/database/free:latest
+```
+
+To launch a container, the needed configuration is the `ORACLE_PWD` of the `SYS` user. You can also map the default 1521 port to your computer.
+
+```bash
+docker run -d --name oracle -p 1521:1521 -e ORACLE_PWD=oracle container-registry.oracle.com/database/free:latest
+```
+
+To connect, you can use a jdbc driver with the URL `jdbc:oracle:thin:<user>/<password>@//localhost:1521/FREE`
+
+### Setup a user
+
+Create a user
+
+```sql
+CREATE USER "C##AZIMUTT" IDENTIFIED BY "azimutt";
+```
+
+Grand permissions
+
+```sql
+GRANT CONNECT, RESOURCE, DBA TO "C##AZIMUTT";
+```
+
+Update user quota on `Users` tablespace
+
+```sql
+ALTER USER "C##AZIMUTT" QUOTA UNLIMITED ON USERS;
+```
+
+### Create a table
+
+```sql
+CREATE TABLE "C##AZIMUTT"."USERS"(
+    user_id NUMBER GENERATED BY DEFAULT AS IDENTITY,
+    first_name VARCHAR2(50) NOT NULL,
+    last_name VARCHAR2(50) NOT NULL,
+    PRIMARY KEY(user_id)
+);
+```

libs/connector-oracle/jest.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+    transform: {'^.+\\.ts?$': 'ts-jest'},
+    testEnvironment: 'node',
+    testRegex: '/src/.+\\.test?\\.(ts|tsx)$',
+    moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node']
+}

libs/connector-oracle/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@azimutt/connector-oracle",
+  "version": "0.1.0",
+  "description": "Connect to Oracle, extract schema, run analysis and queries",
+  "keywords": [],
+  "homepage": "https://azimutt.app",
+  "author": {
+    "name": "Anthony Ly",
+    "email": "[email protected]",
+    "url": "https://anthonyly.dev"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/azimuttapp/azimutt.git",
+    "directory": "libs/connector-oracle"
+  },
+  "main": "./out/index.js",
+  "types": "./out/index.d.ts",
+  "scripts": {
+    "test": "jest",
+    "build": "rm -rf ./out && tsc",
+    "build:docker": "npx tsc",
+    "dry-publish": "pnpm run build && pnpm test && pnpm pack"
+  },
+  "dependencies": {
+    "@azimutt/models": "workspace:^",
+    "@azimutt/utils": "workspace:^",
+    "oracledb": "^6.5.1"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@types/jest": "^29.5.12",
+    "@types/node": "^20.14.5",
+    "@types/oracledb": "^6.5.1",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.3",
+    "typescript": "^5.4.5"
+  }
+}

libs/connector-oracle/src/connect.test.ts

@@ -0,0 +1,30 @@
+import { describe, test } from "@jest/globals"
+import { parseDatabaseUrl } from "@azimutt/models"
+import { connect } from "./connect"
+import { execQuery } from "./query"
+import { application, logger } from "./constants.test"
+
+// Use this test to troubleshoot database connection errors.
+// If you don't succeed with the first one (Azimutt `connect`), try with the second one (raw node lib) and once you found a way, tell us how to fix ;)
+// Of course, you can contact us (issues or [email protected]) to do it together.
+// More documentation available at: https://azimutt.notion.site/Database-connection-troubleshooting-c4c19ed28c7040ef9aaaeec96ce6ba8d
+describe("connect", () => {
+  // TODO 1: replace this with your own connection string, but don't commit it!
+  const url = "jdbc:oracle:thin:sys/oracle@//localhost:1521/FREE"
+
+  // TODO 2: write a valid query for your database
+  const query = "SELECT * FROM C##AZIMUTT.USERS"
+  const parameters: any[] = []
+
+  // TODO 3: unskip this test first and run it (`npm run test -- src/connect.test.ts`)
+  test.skip("Azimutt should connect", async () => {
+    const parsedUrl = parseDatabaseUrl(url)
+    const results = await connect(
+      application,
+      parsedUrl,
+      execQuery(query, parameters),
+      { logger, logQueries: true }
+    )
+    console.log("results", results)
+  })
+})

libs/connector-oracle/src/connect.ts

@@ -0,0 +1,130 @@
+import {
+  Connection,
+  ConnectionAttributes,
+  getConnection,
+  SYSDBA,
+} from "oracledb"
+import { AnyError, errorToString } from "@azimutt/utils"
+import {
+  AttributeValue,
+  ConnectorDefaultOpts,
+  DatabaseUrlParsed,
+  logQueryIfNeeded,
+  queryError,
+} from "@azimutt/models"
+
+export async function connect<T>(
+  application: string,
+  url: DatabaseUrlParsed,
+  exec: (c: Conn) => Promise<T>,
+  opts: ConnectorDefaultOpts
+): Promise<T> {
+  const client = await createConnection(buildConfig(application, url)).catch(
+    (err) => Promise.reject(connectionError(err))
+  )
+  let queryCpt = 1
+  const conn: Conn = {
+    url,
+    query<T extends QueryResultRow>(
+      sql: string,
+      parameters: [] = [],
+      name?: string
+    ): Promise<T[]> {
+      return logQueryIfNeeded(
+        queryCpt++,
+        name,
+        sql,
+        parameters,
+        (sql, parameters) => {
+          return client.execute<T>(sql, parameters).then(
+            (res) => res.rows ?? [],
+            (err) => Promise.reject(queryError(name, sql, err))
+          )
+        },
+        (r) => r?.length ?? 0,
+        opts
+      )
+    },
+    queryArrayMode(
+      sql: string,
+      parameters: any[] = [],
+      name?: string
+    ): Promise<QueryResultArrayMode> {
+      return logQueryIfNeeded(
+        queryCpt++,
+        name,
+        sql,
+        parameters,
+        (sql, parameters) => {
+          return client.execute(sql, parameters).then(
+            (res) => {
+              const { metaData, rows } = res
+              const fields = metaData?.map((meta) => ({
+                name: meta.name,
+              }))
+              return { fields: fields ?? [], rows: (rows as any[]) ?? [] }
+            },
+            (err) => Promise.reject(queryError(name, sql, err))
+          )
+        },
+        (r) => r.rows.length,
+        opts
+      )
+    },
+  }
+  return exec(conn).then(
+    (res) => client.close().then((_) => res),
+    (err) => client.close().then((_) => Promise.reject(err))
+  )
+}
+
+export interface Conn {
+  url: DatabaseUrlParsed
+
+  query<T extends QueryResultRow>(
+    sql: string,
+    parameters?: any[],
+    name?: string
+  ): Promise<T[]>
+
+  queryArrayMode(
+    sql: string,
+    parameters?: any[],
+    name?: string
+  ): Promise<QueryResultArrayMode>
+}
+
+export type QueryResultValue = AttributeValue
+export type QueryResultRow = QueryResultValue[]
+export type QueryResultField = {
+  name: string
+}
+export type QueryResultRowArray = QueryResultValue[]
+export type QueryResultArrayMode = {
+  fields: QueryResultField[]
+  rows: QueryResultRowArray[]
+}
+
+async function createConnection(
+  config: ConnectionAttributes
+): Promise<Connection> {
+  const client = await getConnection(config)
+  return client
+}
+
+function buildConfig(
+  application: string,
+  url: DatabaseUrlParsed
+): ConnectionAttributes {
+  return {
+    connectionString: `${url.host}:${url.port}/${url.db}`,
+    user: url.user,
+    password: url.pass || undefined,
+    privilege: SYSDBA,
+  }
+}
+
+function connectionError(err: AnyError): AnyError {
+  const msg = errorToString(err)
+  return err
+}

libs/connector-oracle/src/constants.test.ts

@@ -0,0 +1,14 @@
+import {expect, test} from "@jest/globals";
+import {Logger} from "@azimutt/utils";
+
+export const logger: Logger = {
+    debug: (text: string): void => console.debug(text),
+    log: (text: string): void => console.log(text),
+    warn: (text: string): void => console.warn(text),
+    error: (text: string): void => console.error(text)
+}
+export const application = 'azimutt-tests'
+
+test('dummy', () => {
+    expect(application).toEqual('azimutt-tests')
+})

libs/connector-oracle/src/helpers.test.ts

@@ -0,0 +1,16 @@
+import { describe, expect, test } from "@jest/globals"
+import { buildSqlColumn, buildSqlTable } from "./helpers"
+
+describe("helpers", () => {
+  test("buildSqlTable", () => {
+    expect(buildSqlTable({ entity: "events" })).toEqual(`"events"`)
+    expect(buildSqlTable({ schema: "", entity: "events" })).toEqual(`"events"`)
+    expect(buildSqlTable({ schema: "public", entity: "events" })).toEqual(
+      `"public"."events"`
+    )
+  })
+  test("buildSqlColumn", () => {
+    expect(buildSqlColumn(["name"])).toEqual(`"name"`)
+    expect(buildSqlColumn(["data", "email"])).toEqual(`"data"->'email'`)
+  })
+})

libs/connector-oracle/src/helpers.ts

@@ -0,0 +1,11 @@
+import { AttributePath, EntityRef, SqlFragment } from "@azimutt/models"
+
+export function buildSqlTable(ref: EntityRef): SqlFragment {
+  const sqlSchema = ref.schema ? `"${ref.schema}".` : ""
+  return `${sqlSchema}"${ref.entity}"`
+}
+
+export function buildSqlColumn(path: AttributePath): SqlFragment {
+  const [head, ...tail] = path
+  return `"${head}"${tail.map((t) => `->'${t}'`).join("")}`
+}