Add support for the new tables sources and compiled_contracts_sources

services/database/migrations/20240729112258-compiled_contracts-constraints.js

@@ -107,11 +107,16 @@ exports.down = function (db, callback) {
         ALTER TABLE compiled_contracts
         ADD CONSTRAINT compiled_contracts_pseudo_pkey UNIQUE (compiler, language, creation_code_hash, runtime_code_hash);
 
-        DROP FUNCTION IF EXISTS validate_json_object_keys;
-        DROP FUNCTION IF EXISTS validate_json_object_keys;
-        DROP FUNCTION IF EXISTS validate_compilation_artifacts;
-        DROP FUNCTION IF EXISTS validate_creation_code_artifacts;
+        ALTER TABLE compiled_contracts
+        DROP CONSTRAINT IF EXISTS compilation_artifacts_object,
+        DROP CONSTRAINT IF EXISTS creation_code_artifacts_object,
+        DROP CONSTRAINT IF EXISTS runtime_code_artifacts_object;
+
         DROP FUNCTION IF EXISTS validate_runtime_code_artifacts;
+        DROP FUNCTION IF EXISTS validate_creation_code_artifacts;
+        DROP FUNCTION IF EXISTS validate_compilation_artifacts;
+        DROP FUNCTION IF EXISTS validate_json_object_keys(jsonb, text[], text[]);
+        DROP FUNCTION IF EXISTS validate_json_object_keys(jsonb, text[]);
         `,
       ),
     ],

services/database/migrations/20240918154734-add-sources-table.js

@@ -0,0 +1,90 @@
+"use strict";
+
+var async = require("async");
+
+var dbm;
+var type;
+var seed;
+
+/**
+ * We receive the dbmigrate dependency from dbmigrate initially.
+ * This enables us to not have to rely on NODE_PATH.
+ */
+exports.setup = function (options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+};
+
+exports.up = function (db, callback) {
+  async.series(
+    [
+      db.runSql.bind(
+        db,
+        `
+          CREATE TABLE sources
+          (
+              /* the sha256 hash of the source code */
+              source_hash bytea NOT NULL PRIMARY KEY,
+
+              /* the keccak256 hash of the source code */
+              source_hash_keccak bytea NOT NULL,
+
+              /* the actual source code content */
+              content varchar NOT NULL,
+
+              /* timestamps */
+              created_at  timestamptz NOT NULL DEFAULT NOW(),
+              updated_at  timestamptz NOT NULL DEFAULT NOW(),
+
+              /* ownership */
+              created_by  varchar NOT NULL DEFAULT (current_user),
+              updated_by  varchar NOT NULL DEFAULT (current_user),
+
+              CONSTRAINT source_hash_check CHECK (source_hash = digest(content, 'sha256'))
+          );
+
+          CREATE TABLE compiled_contracts_sources
+          (
+              id uuid NOT NULL PRIMARY KEY DEFAULT gen_random_uuid(),
+
+              /* the specific compilation and the specific source */
+              compilation_id uuid NOT NULL REFERENCES compiled_contracts(id),
+              source_hash bytea NOT NULL REFERENCES sources(source_hash),
+
+              /* the file path associated with this source code in the compilation */
+              path varchar NOT NULL,
+
+              CONSTRAINT compiled_contracts_sources_pseudo_pkey UNIQUE (compilation_id, path)
+          );
+
+          CREATE INDEX compiled_contracts_sources_source_hash ON compiled_contracts_sources USING btree (source_hash);
+          CREATE INDEX compiled_contracts_sources_compilation_id ON compiled_contracts_sources (compilation_id);
+
+          ALTER TABLE compiled_contracts DROP COLUMN sources;
+        `,
+      ),
+    ],
+    callback,
+  );
+};
+
+exports.down = function (db, callback) {
+  async.series(
+    [
+      db.runSql.bind(
+        db,
+        `
+      DROP TABLE compiled_contracts_sources;
+      DROP TABLE sources;
+      ALTER TABLE compiled_contracts ADD COLUMN sources jsonb NOT NULL;
+    `,
+      ),
+    ],
+    callback,
+  );
+};
+
+exports._meta = {
+  version: 1,
+};

services/server/src/server/controllers/repository/repository.routes.ts

@@ -136,8 +136,8 @@ router
 
 /**
  * The following two routes are the replacement for the removed static file route that exposed RepositoryV1
- * The function getFileEndpoint get the sources from compiled_contracts.sources
- * We need both of these routes because compiled_contracts.sources doesn't contain the metadata file
+ * The function getFileEndpoint get the sources from compiled_contracts_sources
+ * We need both of these routes because compiled_contracts_sources doesn't contain the metadata file
  */
 
 // This route covers constructor-args.txt, creator-tx-hash.txt, library-map.json, immutable-references.json files

services/server/src/server/services/storageServices/AbstractDatabaseService.ts

@@ -301,6 +301,18 @@ export default abstract class AbstractDatabaseService {
       };
     }
 
+    const sourcesInformation = Object.keys(recompiledContract.solidity).map(
+      (path) => {
+        return {
+          path,
+          source_hash_keccak: bytesFromString<BytesKeccak>(
+            keccak256(Buffer.from(recompiledContract.solidity[path])),
+          ),
+          content: recompiledContract.solidity[path],
+        };
+      },
+    );
+
     return {
       recompiledCreationCode,
       recompiledRuntimeCode: {
@@ -331,13 +343,13 @@ export default abstract class AbstractDatabaseService {
         compiler: "solc",
         compiler_settings: Database.prepareCompilerSettings(recompiledContract),
         name: recompiledContract.name,
-        sources: recompiledContract.solidity,
         version: recompiledContract.compilerVersion,
         fully_qualified_name: `${compilationTargetPath}:${compilationTargetName}`,
         compilation_artifacts: compilationArtifacts,
         creation_code_artifacts: creationCodeArtifacts,
         runtime_code_artifacts: runtimeCodeArtifacts,
       },
+      sourcesInformation,
       verifiedContract: {
         runtime_transformations,
         creation_transformations,
@@ -357,7 +369,12 @@ export default abstract class AbstractDatabaseService {
     match: Match,
     databaseColumns: Database.DatabaseColumns,
   ): Promise<number> {
+    // Get a client from the pool, so that we can execute all the insert queries within the same transaction
+    const client = await this.databasePool.connect();
+
     try {
+      // Start the sql transaction
+      await client.query("BEGIN");
       let recompiledCreationCodeInsertResult:
         | QueryResult<Pick<Database.Tables.Code, "bytecode_hash">>
         | undefined;
@@ -368,67 +385,77 @@ export default abstract class AbstractDatabaseService {
       // Add recompiled bytecodes
       if (databaseColumns.recompiledCreationCode) {
         recompiledCreationCodeInsertResult = await Database.insertCode(
-          this.databasePool,
+          client,
           databaseColumns.recompiledCreationCode,
         );
       }
       const recompiledRuntimeCodeInsertResult = await Database.insertCode(
-        this.databasePool,
+        client,
         databaseColumns.recompiledRuntimeCode,
       );
 
       // Add onchain bytecodes
       if (databaseColumns.onchainCreationCode) {
         onchainCreationCodeInsertResult = await Database.insertCode(
-          this.databasePool,
+          client,
           databaseColumns.onchainCreationCode,
         );
       }
       const onchainRuntimeCodeInsertResult = await Database.insertCode(
-        this.databasePool,
+        client,
         databaseColumns.onchainRuntimeCode,
       );
 
       // Add the onchain contract in contracts
-      const contractInsertResult = await Database.insertContract(
-        this.databasePool,
-        {
-          creation_bytecode_hash:
-            onchainCreationCodeInsertResult?.rows[0].bytecode_hash,
-          runtime_bytecode_hash:
-            onchainRuntimeCodeInsertResult.rows[0].bytecode_hash,
-        },
-      );
+      const contractInsertResult = await Database.insertContract(client, {
+        creation_bytecode_hash:
+          onchainCreationCodeInsertResult?.rows[0].bytecode_hash,
+        runtime_bytecode_hash:
+          onchainRuntimeCodeInsertResult.rows[0].bytecode_hash,
+      });
 
       // add the onchain contract in contract_deployments
       const contractDeploymentInsertResult =
-        await Database.insertContractDeployment(this.databasePool, {
+        await Database.insertContractDeployment(client, {
           ...databaseColumns.contractDeployment,
           contract_id: contractInsertResult.rows[0].id,
         });
 
       // insert new recompiled contract
       const compiledContractsInsertResult =
-        await Database.insertCompiledContract(this.databasePool, {
+        await Database.insertCompiledContract(client, {
           ...databaseColumns.compiledContract,
           creation_code_hash:
             recompiledCreationCodeInsertResult?.rows[0].bytecode_hash,
           runtime_code_hash:
             recompiledRuntimeCodeInsertResult.rows[0].bytecode_hash,
         });
 
+      const compiledContractId = compiledContractsInsertResult.rows[0].id;
+
+      await Database.insertCompiledContractsSources(client, {
+        sourcesInformation: databaseColumns.sourcesInformation,
+        compilation_id: compiledContractId,
+      });
+
       // insert new recompiled contract with newly added contract and compiledContract
       const verifiedContractInsertResult =
-        await Database.insertVerifiedContract(this.databasePool, {
+        await Database.insertVerifiedContract(client, {
           ...databaseColumns.verifiedContract,
-          compilation_id: compiledContractsInsertResult.rows[0].id,
+          compilation_id: compiledContractId,
           deployment_id: contractDeploymentInsertResult.rows[0].id,
         });
+      // Commit the transaction
+      await client.query("COMMIT");
       return verifiedContractInsertResult.rows[0].id;
     } catch (e) {
+      // Rollback the transaction in case of error
+      await client.query("ROLLBACK");
       throw new Error(
         `cannot insert verified_contract address=${match.address} chainId=${match.chainId}\n${e}`,
       );
+    } finally {
+      client.release();
     }
   }
 
@@ -446,7 +473,12 @@ export default abstract class AbstractDatabaseService {
       throw new Error("Missing onchain runtime bytecode");
     }
 
+    // Get a client from the pool, so that we can execute all the insert queries within the same transaction
+    const client = await this.databasePool.connect();
     try {
+      // Start the sql transaction
+      await client.query("BEGIN");
+
       let recompiledCreationCodeInsertResult:
         | QueryResult<Pick<Database.Tables.Code, "bytecode_hash">>
         | undefined;
@@ -460,28 +492,25 @@ export default abstract class AbstractDatabaseService {
         databaseColumns.onchainCreationCode
       ) {
         onchainCreationCodeInsertResult = await Database.insertCode(
-          this.databasePool,
+          client,
           databaseColumns.onchainCreationCode,
         );
 
         const onchainRuntimeCodeInsertResult = await Database.insertCode(
-          this.databasePool,
+          client,
           databaseColumns.onchainRuntimeCode,
         );
 
         // Add the onchain contract in contracts
-        const contractInsertResult = await Database.insertContract(
-          this.databasePool,
-          {
-            creation_bytecode_hash:
-              onchainCreationCodeInsertResult.rows[0].bytecode_hash,
-            runtime_bytecode_hash:
-              onchainRuntimeCodeInsertResult.rows[0].bytecode_hash,
-          },
-        );
+        const contractInsertResult = await Database.insertContract(client, {
+          creation_bytecode_hash:
+            onchainCreationCodeInsertResult.rows[0].bytecode_hash,
+          runtime_bytecode_hash:
+            onchainRuntimeCodeInsertResult.rows[0].bytecode_hash,
+        });
 
         // add the onchain contract in contract_deployments
-        await Database.updateContractDeployment(this.databasePool, {
+        await Database.updateContractDeployment(client, {
           ...databaseColumns.contractDeployment,
           contract_id: contractInsertResult.rows[0].id,
           id: existingVerifiedContractResult[0].deployment_id,
@@ -494,18 +523,18 @@ export default abstract class AbstractDatabaseService {
         databaseColumns.recompiledCreationCode
       ) {
         recompiledCreationCodeInsertResult = await Database.insertCode(
-          this.databasePool,
+          client,
           databaseColumns.recompiledCreationCode,
         );
       }
       const recompiledRuntimeCodeInsertResult = await Database.insertCode(
-        this.databasePool,
+        client,
         databaseColumns.recompiledRuntimeCode,
       );
 
       // insert new recompiled contract
       const compiledContractsInsertResult =
-        await Database.insertCompiledContract(this.databasePool, {
+        await Database.insertCompiledContract(client, {
           ...databaseColumns.compiledContract,
           creation_code_hash:
             recompiledCreationCodeInsertResult?.rows[0].bytecode_hash,
@@ -515,17 +544,23 @@ export default abstract class AbstractDatabaseService {
 
       // update verified contract with the newly added recompiled contract
       const verifiedContractInsertResult =
-        await Database.insertVerifiedContract(this.databasePool, {
+        await Database.insertVerifiedContract(client, {
           ...databaseColumns.verifiedContract,
           compilation_id: compiledContractsInsertResult.rows[0].id,
           deployment_id: existingVerifiedContractResult[0].deployment_id,
         });
 
+      // Commit the transaction
+      await client.query("COMMIT");
       return verifiedContractInsertResult.rows[0].id;
     } catch (e) {
+      // Rollback the transaction in case of error
+      await client.query("ROLLBACK");
       throw new Error(
         `cannot update verified_contract address=${match.address} chainId=${match.chainId}\n${e}`,
       );
+    } finally {
+      client.release();
     }
   }
 

services/server/src/server/services/storageServices/SourcifyDatabaseService.ts

@@ -254,7 +254,7 @@ export class SourcifyDatabaseService
   };
 
   /**
-   * getFiles extracts the files from the database `compiled_contracts.sources`
+   * getFiles extracts the files from the database `compiled_contracts_sources`
    * and store them into FilesInfo.files, this object is then going to be formatted
    * by getTree, getContent and getFile.
    */
@@ -281,12 +281,18 @@ export class SourcifyDatabaseService
         ? "full"
         : "partial";
 
-    const sources: { [index: string]: string } = {};
-
-    // Add 'sources/' prefix for API compatibility with the repoV1 responses. RepoV1 filesystem has all source files in 'sources/'
-    for (const path of Object.keys(sourcifyMatch.sources)) {
-      sources[`sources/${path}`] = sourcifyMatch.sources[path];
-    }
+    const sourcesResult = await Database.getCompiledContractSources(
+      this.databasePool,
+      sourcifyMatch.compilation_id,
+    );
+    const sources = sourcesResult.rows.reduce(
+      (sources, source) => {
+        // Add 'sources/' prefix for API compatibility with the repoV1 responses. RepoV1 filesystem has all source files in 'sources/'
+        sources[`sources/${source.path}`] = source.content;
+        return sources;
+      },
+      {} as Record<string, string>,
+    );
     const files: FilesRawValue = {};
 
     if (sourcifyMatch.metadata) {