-
Notifications
You must be signed in to change notification settings - Fork 3
/
addCsv.js
121 lines (105 loc) · 5.46 KB
/
addCsv.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
const csv = require('csv')
const fs = require('fs')
const { PromisePool } = require('@supercharge/promise-pool')
const { setKey, initDB } = require("./db")
// source: https://github.com/bodino/Fndair_backend/tree/main/Backend/Data
// source: https://github.com/rotki/data/tree/main/airdrops
const csvConfigs_done = [
{ file: 'omni.csv', decimals: 0, key: 'omni', addressField: 'address', valueField: 'amount' },
{ file: '1inch.csv', decimals: 0, key: '1inch', addressField: 'address', valueField: 'amount' },
{ file: 'convex_airdrop.csv', decimals: 0, key: 'convex', addressField: 'address', valueField: 'amount' },
{ file: 'cornichon_airdrop.csv', decimals: 0, key: 'cornichon', addressField: 'address', valueField: 'amount' },
{ file: 'cow_gnosis.csv', decimals: 18, key: 'cow0', addressField: 'address', valueField: 'amount' },
{ file: 'cow_mainnet.csv', decimals: 18, key: 'cow1', addressField: 'address', valueField: 'amount' },
{ file: 'curve_airdrop.csv', decimals: 0, key: 'curve', addressField: 'address', valueField: 'amount' },
{ file: 'degen2_season1.csv', decimals: 0, key: 'degen', addressField: 'address', valueField: 'amount' },
{ file: 'diva.csv', decimals: 0, key: 'diva', addressField: 'address', valueField: 'amount' },
{ file: 'ens.csv', decimals: 0, key: 'ens', addressField: 'address', valueField: 'amount' },
{ file: 'furucombo_airdrop.csv', decimals: 0, key: 'furucombo', addressField: 'address', valueField: 'amount' },
{ file: 'grain_iou.csv', decimals: 18, key: 'grain', addressField: 'address', valueField: 'amount' },
{ file: 'lido_airdrop.csv', decimals: 18, key: 'lido', addressField: 'address', valueField: 'amount' },
{ file: 'optimism_4.csv', decimals: 0, key: 'op', addressField: 'address', valueField: 'amount' },
{ file: 'psp.csv', decimals: 0, key: 'psp', addressField: 'address', valueField: 'amount' },
// { file: 'shapeshift.csv', decimals: 0, key: 'shapeshift', addressField: 'address', valueField: 'amount' }, // too big, skipped it
{ file: 'shutter.csv', decimals: 0, key: 'shutter', addressField: 'address', valueField: 'amount' },
{ file: 'starknet.csv', decimals: 0, key: 'starknet', addressField: 'address', valueField: 'amount' },
{ file: 'saddle_finance.csv', decimals: 18, key: 'saddle', addressField: 'address', valueField: 'amount' },
{ file: 'tornado.csv', decimals: 18, key: 'tornado', addressField: 'address', valueField: 'amount' },
{ file: 'uniswap.csv', decimals: 0, key: 'uniswap', addressField: 'address', valueField: 'amount' },
{ file: 'Dydx.json', decimals: 0, key: 'dydx', },
{ file: 'EUL.json', decimals: 0, key: 'eul', },
{ file: 'Hop.json', decimals: 0, key: 'hop', },
{ file: 'kmno_allocation_v3.csv', decimals: 0, key: 'kamino', addressField: 'wallet', valueField: 'kmno_allocated' },
{ file: 'myso-airdrop-prod.json', decimals: 18, key: 'myso', },
{ file: 'jup_allocation_final.csv', decimals: 0, key: 'jup', addressField: 'pubkey', valueField: 'amount' },
{ file: 'ekubo.csv', decimals: 0, key: 'ekubo', },
{ file: 'zk.csv', decimals: 0, key: 'zk', addressField: 'userId', valueField: 'tokenAmount' },
{ file: 'debridge.csv', decimals: 0, key: 'db1', addressField: 'userId', valueField: 'dist1' },
]
const csvConfigs = [
{ file: 'debridge.csv', decimals: 0, key: 'db2', addressField: 'userId', valueField: 'dist2' },
]
async function addCsv() {
initDB()
for (const config of csvConfigs) {
const csvData = await readCSV(config)
let { decimals, key, addressField, valueField } = config
if (!addressField) {
console.info(key, 'assuming first column is address field, and the next is value')
addressField = 0
valueField = 1
} else {
const headRow = csvData.shift()
addressField = headRow.indexOf(addressField)
valueField = headRow.indexOf(valueField)
}
const timeKey = `${key} took`
console.time(timeKey)
console.info(`Adding ${csvData.length} records for`, key)
let i = 0
await PromisePool.withConcurrency(100)
.for(csvData)
.process(async (record, idx) => {
if (idx % 10000 === 0) console.info('Processed', Number(100 * idx / csvData.length).toFixed(2), '%')
const address = record[addressField]
let value = record[valueField]
if (!value) return;
// if (++i < 15)
// console.log('address', address, 'value', value)
// return;
if (isNaN(+value)) {
console.info('Invalid value for key', address, key, value)
return;
}
if (decimals || decimals === 0) value = Math.round(value / 10 ** decimals)
if (value < 1) return;
await setKey(address, key, value)
})
console.timeEnd(timeKey)
}
}
async function readCSV({ file, delimiter = ',', key }) {
const isJson = file.endsWith('.json')
const csvFolder = './csv-data/'
if (!isJson) {
const csvData = []
return new Promise((resolve) =>
fs.createReadStream(csvFolder + file)
.pipe(csv.parse({ delimiter }))
.on('data', row => csvData.push(row))
.on('end', () => resolve(csvData)))
}
const data = require(csvFolder + file)
switch (key) {
case 'myso':
return Object.entries(data).map(([address, { amount }]) => [address, amount])
}
if (typeof data.Data === 'object' && typeof data.Info === 'object') {
return Object.entries(data.Data).map(([address, { tokens }]) => [address, tokens])
}
throw new Error('Not implemented!')
}
addCsv().catch(console.error).then(() => {
console.log('done')
process.exit(0)
})