diff --git a/deploy/base/clickhouse/clickhouse.cue b/deploy/base/clickhouse/clickhouse.cue new file mode 100644 index 0000000..3661ec6 --- /dev/null +++ b/deploy/base/clickhouse/clickhouse.cue @@ -0,0 +1,54 @@ +package clickhouse + +import ( + schema "github.com/monogon-dev/NetMeta/deploy/single-node/schema" + "netmeta.monogon.tech/xml" +) + +// A stripped down version of the #SamplerConfig found in deploy/single-node/config.cue +#SamplerConfig: [string]: { + device: string + samplingRate: int + anonymizeAddresses: bool + description: string + interface: [string]: { + id: int + description: string + } + vlan: [string]: { + id: int + description: string + } + host: [string]: { + device: string + description: string + } + ... +} + +#UserData: autnums: [string]: { + asn: int + name: string + country: string +} + +#Config: { + sampler: #SamplerConfig + userData: #UserData + + dataPath: string + risinfoURL: string +} + +files: { + // Iterate over our required files from schema, e.g. the protobuf files + for k, v in schema.file { + "\(k)": v + } + + // Iterate over all defined files in _files and generate the config files for clickhouse + for k, v in _files { + "\(k).conf": (xml.#Marshal & {in: v._cfg}).out + "\(k).tsv": v.data + } +} diff --git a/deploy/base/clickhouse/external.cue b/deploy/base/clickhouse/external.cue new file mode 100644 index 0000000..b2c6444 --- /dev/null +++ b/deploy/base/clickhouse/external.cue @@ -0,0 +1,89 @@ +package clickhouse + +import ( + "netmeta.monogon.tech/xml" +) + +files: "risinfo.conf": (xml.#Marshal & {in: yandex: dictionary: { + name: "risinfo" + source: http: { + url: "\(#Config.risinfoURL)/rib.tsv" + format: "TabSeparated" + } + lifetime: 3600 + layout: ip_trie: access_to_key_from_attributes: true + structure: key: attribute: { + name: "prefix" + type: "String" + } + structure: attribute: { + name: "asnum" + type: "UInt32" + null_value: 0 + } +}}).out + +files: "autnums.conf": (xml.#Marshal & {in: yandex: dictionary: { + name: "autnums" + source: clickhouse: query: + #""" + SELECT * FROM dictionaries.risinfo_autnums + UNION ALL + SELECT * FROM dictionaries.user_autnums + """# + lifetime: 3600 + layout: flat: null + structure: [{ + id: name: "asnum" + }, { + attribute: { + name: "name" + type: "String" + null_value: null + } + }, { + attribute: { + name: "country" + type: "String" + null_value: null + } + }] +}}).out + +files: "risinfo_autnums.conf": (xml.#Marshal & {in: yandex: dictionary: { + name: "risinfo_autnums" + source: http: { + url: "\(#Config.risinfoURL)/autnums.tsv" + format: "TabSeparated" + } + lifetime: 86400 + layout: flat: null + structure: [{ + id: name: "asnum" + }, { + attribute: { + name: "name" + type: "String" + null_value: null + } + }, { + attribute: { + name: "country" + type: "String" + null_value: null + } + }] +}}).out + +files: "format_function.xml": (xml.#Marshal & {in: yandex: functions: { + type: "executable" + name: "formatQuery" + return_type: "String" + argument: [{ + type: "String" + name: "query" + }] + format: "LineAsString" + command: "clickhouse format --oneline" + execute_direct: "0" +}}).out diff --git a/deploy/single-node/k8s/clickhouse/files.cue b/deploy/base/clickhouse/files.cue similarity index 52% rename from deploy/single-node/k8s/clickhouse/files.cue rename to deploy/base/clickhouse/files.cue index bd9aa81..4eaa094 100644 --- a/deploy/single-node/k8s/clickhouse/files.cue +++ b/deploy/base/clickhouse/files.cue @@ -3,7 +3,6 @@ package clickhouse import ( "strings" "strconv" - "netmeta.monogon.tech/xml" ) // template for TSV dictionaries @@ -20,7 +19,7 @@ _files: [NAME=string]: { name: NAME source: [{ file: { - path: "/etc/clickhouse-server/config.d/\(NAME).tsv" + path: "\(#Config.dataPath)/\(NAME).tsv" format: "TSV" } settings: format_tsv_null_representation: "NULL" @@ -29,17 +28,9 @@ _files: [NAME=string]: { } } -// Iterate over all defined files in _files and generate the config files for clickhouse -ClickHouseInstallation: netmeta: spec: configuration: files: { - for k, v in _files { - "\(k).conf": (xml.#Marshal & {in: v._cfg}).out - "\(k).tsv": v.data - } -} - // Dictionary for user-defined interface name lookup _files: InterfaceNames: { - data: strings.Join([ for s in #Config.sampler for i in s.interface { + data: strings.Join([for s in #Config.sampler for i in s.interface { strings.Join([s.device, "\(i.id)", i.description], "\t") }], "\n") @@ -68,7 +59,7 @@ _files: InterfaceNames: { // Dictionary for user-defined sampler settings lookup _files: SamplerConfig: { - data: strings.Join([ for s in #Config.sampler { + data: strings.Join([for s in #Config.sampler { let samplingRate = [ if s.samplingRate == 0 { "NULL" @@ -119,7 +110,7 @@ _files: SamplerConfig: { // Dictionary for user-defined vlan name lookup _files: VlanNames: { - data: strings.Join([ for s in #Config.sampler for v in s.vlan { + data: strings.Join([for s in #Config.sampler for v in s.vlan { strings.Join([s.device, "\(v.id)", v.description], "\t") }], "\n") @@ -148,7 +139,7 @@ _files: VlanNames: { // Dictionary for user-defined host name lookup _files: HostNames: { - data: strings.Join([ for s in #Config.sampler for h in s.host { + data: strings.Join([for s in #Config.sampler for h in s.host { strings.Join([s.device, h.device, h.description], "\t") }], "\n") @@ -176,7 +167,7 @@ _files: HostNames: { } _files: user_autnums: { - data: strings.Join([ for _, e in #Config.userData.autnums { + data: strings.Join([for _, e in #Config.userData.autnums { strings.Join(["\(e.asn)", e.name, e.country], "\t") }], "\n") @@ -199,97 +190,3 @@ _files: user_autnums: { }] } } - -ClickHouseInstallation: netmeta: spec: configuration: files: "risinfo.conf": (xml.#Marshal & {in: { - yandex: dictionary: { - name: "risinfo" - source: http: { - url: "http://risinfo/rib.tsv" - format: "TabSeparated" - } - lifetime: 3600 - layout: ip_trie: access_to_key_from_attributes: true - structure: key: attribute: { - name: "prefix" - type: "String" - } - structure: attribute: { - name: "asnum" - type: "UInt32" - null_value: 0 - } - } -}}).out - -ClickHouseInstallation: netmeta: spec: configuration: files: "autnums.conf": (xml.#Marshal & {in: { - yandex: dictionary: { - name: "autnums" - source: clickhouse: { - query: - #""" - SELECT * FROM dictionaries.risinfo_autnums - UNION ALL - SELECT * FROM dictionaries.user_autnums - """# - } - lifetime: 3600 - layout: flat: null - structure: [{ - id: name: "asnum" - }, { - attribute: { - name: "name" - type: "String" - null_value: null - } - }, { - attribute: { - name: "country" - type: "String" - null_value: null - } - }] - } -}}).out - -ClickHouseInstallation: netmeta: spec: configuration: files: "risinfo_autnums.conf": (xml.#Marshal & {in: { - yandex: dictionary: { - name: "risinfo_autnums" - source: http: { - url: "http://risinfo/autnums.tsv" - format: "TabSeparated" - } - lifetime: 86400 - layout: flat: null - structure: [{ - id: name: "asnum" - }, { - attribute: { - name: "name" - type: "String" - null_value: null - } - }, { - attribute: { - name: "country" - type: "String" - null_value: null - } - }] - } -}}).out - -ClickHouseInstallation: netmeta: spec: configuration: files: "format_function.xml": (xml.#Marshal & {in: { - yandex: functions: { - type: "executable" - name: "formatQuery" - return_type: "String" - argument: [{ - type: "String" - name: "query" - }] - format: "LineAsString" - command: "clickhouse format --oneline" - execute_direct: "0" - } -}}).out diff --git a/deploy/single-node/k8s/clickhouse/static_files.cue b/deploy/base/clickhouse/static_files.cue similarity index 95% rename from deploy/single-node/k8s/clickhouse/static_files.cue rename to deploy/base/clickhouse/static_files.cue index 82bbf34..b5aee5a 100644 --- a/deploy/single-node/k8s/clickhouse/static_files.cue +++ b/deploy/base/clickhouse/static_files.cue @@ -1,9 +1,5 @@ package clickhouse -import ( - schema "github.com/monogon-dev/NetMeta/deploy/single-node/schema" -) - // curl -s https://www.iana.org/assignments/protocol-numbers/protocol-numbers-1.csv | awk -F ',' '{ print $1 "\t" $2 }' // plus some manual post-processing. Most of them are never seen on the wire, but doesn't hurt to have the full list. _files: IPProtocols: cfg: { @@ -252,9 +248,3 @@ _files: TCPFlags: data: #""" 64 ECE 128 CWR """# - -ClickHouseInstallation: netmeta: spec: configuration: files: { - for k, v in schema.file { - "\(k)": v - } -} \ No newline at end of file diff --git a/deploy/base/reconciler/reconciler.cue b/deploy/base/reconciler/reconciler.cue new file mode 100644 index 0000000..40d6f7e --- /dev/null +++ b/deploy/base/reconciler/reconciler.cue @@ -0,0 +1,18 @@ +package reconciler + +import ( + reconciler "github.com/monogon-dev/NetMeta/reconciler:main" + "encoding/json" +) + +#Config: { + files: [string]: string + config: reconciler.#Config +} + +files: { + for name, content in #Config.files { + "\(name)": content + } + "config.json": json.Marshal(#Config.config) +} diff --git a/deploy/nix/config.cue b/deploy/nix/config.cue new file mode 100644 index 0000000..264feda --- /dev/null +++ b/deploy/nix/config.cue @@ -0,0 +1,122 @@ +package nix + +import ( + "net" + "strconv" +) + +#DashboardDisplayConfig: { + // Minimum interval for all panels. By default, there's no minimum interval and the interval goes all the way down + // to minimum resolution (1s). For IPFIX, the minimum flow export interval may be a large multiple of that, + // resulting in misleading rendering when zooming in (spikes at multiples of the flow export resolution). + // + // For IPFIX, set minInterval to 2× the minimum flow timeout on the network device. + minInterval: string | null | *null + + // Maximum packet size for heatmap panel. We set a fixed maximum value to filter out spurious oversizes packets from + // loopback interfaces and have the right scale when only small packets are visible. + maxPacketSize: *1500 | uint + + // The config for the FastNetMon integration + fastNetMon: *null | { + // Name of a FastNetMon InfluxDB datasource. If you use NetMeta alongside FastNetMon, attack + // notifications can be shown in NetMeta. You have to manually create the FastNetMon + // datasource and connect it to your instance. + dataSource: string | *"FastNetMon InfluxDB" + } +} + +// IPv6 or pseudo-IPv4 mapped address like ::ffff:100.0.0.1 +#DeviceAddress: string & net.IP & !~"^:?:?[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$" + +// A small test to verify that #DeviceAddress forbids the old IP mapping +_deviceAddressTest: { + IN=in: { + // Key is the value to test + // Value is the parsing result + "2001:0DB8::1": true + "::ffff:127.0.0.1": true + "::ffff:127.0.0.1": true + "::127.0.0.1": false + "127.0.0.1": false + } + + out: { + for k, v in IN { + "\(k)": v + "\(k)": (k & #DeviceAddress) != _|_ + } + } +} + +// for COLUMN you can use the following columns: +// SamplerAddress, SrcAddr, DstAddr, SrcAS, DstAS, FlowDirection +// additionally it is possible to use these with a valid CIDR +// SamplerAddressInRange, SrcAddrInRange, DstAddrInRange +#ColumnExpression: [COLUMN=string]: string | int + +// A struct containing sampler specific config parameters +#SamplerConfig: [DEVICE=string]: { + // Router source address + device: DEVICE & #DeviceAddress + + // Sampling rate to override the sampling rate provided by the sampler + samplingRate: int | *0 + + // Human-readable host description to show in the frontend + description: string | *"" + + // anonymize the last 8byte for v6 addresses and 1byte for v4 addresses + anonymizeAddresses: bool | *false + + // how to detect an incoming flow + // each array entry is connected by an OR statement + // everything inside a #ColumnExpression is connected with AND + isIncomingFlow: [...#ColumnExpression] + + // Interface names for the data from this sampler + interface: [ID=string]: { + // Numeric interface Index (often known as the "SNMP ID") + id: *strconv.Atoi(ID) | int + + // Human-readable interface description to show in the frontend + description: string + } + + vlan: [ID=string]: { + // Numeric VLAN ID + id: *strconv.Atoi(ID) | int + + // Human-readable vlan description to show in the frontend + description: string + } + + // Host names for the data from this sampler + host: [DEVICE=string]: { + // Host source address + device: DEVICE & #DeviceAddress + + // Human-readable host description to show in the frontend + description: string + } +} + +#UserData: { + // Custom ASNs + autnums: [ASN=string]: { + asn: *strconv.Atoi(ASN) | int + name: string + country: string + } +} + +#NetMetaConfig: { + // Dashboard display config - these settings only affect rendering of the Grafana dashboards. + dashboardDisplay: #DashboardDisplayConfig + + // Config parameter like interface names. See #SamplerConfig + sampler: #SamplerConfig + + // Userprovided data like custom ASNs + userData: #UserData +} diff --git a/deploy/nix/defs.cue b/deploy/nix/defs.cue new file mode 100644 index 0000000..a3ae052 --- /dev/null +++ b/deploy/nix/defs.cue @@ -0,0 +1,49 @@ +package nix + +import ( + // Dashboards + grafana_dashboards "github.com/monogon-dev/NetMeta/deploy/dashboards" + schema "github.com/monogon-dev/NetMeta/deploy/single-node/schema" + clickhouse "github.com/monogon-dev/NetMeta/deploy/base/clickhouse" + reconciler "github.com/monogon-dev/NetMeta/deploy/base/reconciler" +) + +netmeta: config: #NetMetaConfig +netmeta: dashboards: { + _dashboards: (grafana_dashboards & { + #Config: netmeta.config.dashboardDisplay + }) + for k, v in _dashboards.dashboards { + "\(k)": v + } +} + +_schema: (schema & { + #Config: { + sampler: netmeta.config.sampler + kafkaBrokerList: "localhost:9092" + } +}) + +out: dashboards: netmeta.dashboards + +out: "clickhouse": (clickhouse & { + #Config: { + sampler: netmeta.config.sampler + userData: netmeta.config.userData + risinfoURL: "foobar" + dataPath: "foobar" + } +}) + +out: "reconciler": (reconciler & { + for name, content in _schema.file { + "\(name)": content + } + "config.json": json.Marshal({ + database: "default" + functions: [for _, v in _schema.function {v}] + materialized_views: [for _, v in _schema.view {v}] + source_tables: [for _, v in _schema.table {v}] + }) +}) diff --git a/deploy/nix/dump_tool.cue b/deploy/nix/dump_tool.cue new file mode 100644 index 0000000..d26210d --- /dev/null +++ b/deploy/nix/dump_tool.cue @@ -0,0 +1,54 @@ +package nix + +import ( + "encoding/yaml" + "encoding/json" + "strings" + "tool/file" +) + +command: dump: { + dashboards: { + outDir: file.MkdirAll & { + path: "out/dashboards" + } + + for k, v in out.dashboards { + let fileName = "\(strings.ToLower(strings.Replace(k, " ", "_", -1))).json" + "\(outDir.path)/\(fileName)": file.Create & { + $after: outDir + filename: "\(outDir.path)/\(fileName)" + contents: json.Indent(json.Marshal(v), "", " ") + } + } + } + + clickhouse: { + outDir: file.MkdirAll & { + path: "out/clickhouse" + } + + for k, v in out.clickhouse.files { + "\(outDir.path)/\(k)": file.Create & { + $after: outDir + filename: "\(outDir.path)/\(k)" + contents: v + } + } + } + + reconciler: { + outDir: file.MkdirAll & { + path: "out/reconciler" + } + + for k, v in out.reconciler.files { + "\(outDir.path)/\(k)": file.Create & { + $after: outDir + filename: "\(outDir.path)/\(k)" + contents: v + } + } + } + +} diff --git a/deploy/single-node/defs.cue b/deploy/single-node/defs.cue index 66db72d..6e51bc7 100644 --- a/deploy/single-node/defs.cue +++ b/deploy/single-node/defs.cue @@ -49,6 +49,7 @@ _schema: (schema & { #Config: { fastNetMon: netmeta.config.fastNetMon sampler: netmeta.config.sampler + kafkaBrokerList: "netmeta-kafka-bootstrap:9092" } }) diff --git a/deploy/single-node/k8s/clickhouse/clickhouse.cue b/deploy/single-node/k8s/clickhouse/clickhouse.cue index e3ff809..d81e368 100644 --- a/deploy/single-node/k8s/clickhouse/clickhouse.cue +++ b/deploy/single-node/k8s/clickhouse/clickhouse.cue @@ -3,43 +3,15 @@ package clickhouse import ( "crypto/sha256" "encoding/hex" + chBase "github.com/monogon-dev/NetMeta/deploy/base/clickhouse" ) -// A stripped down version of the #SamplerConfig found in deploy/single-node/config.cue -#SamplerConfig: [string]: { - device: string - samplingRate: int - anonymizeAddresses: bool - description: string - interface: [string]: { - id: int - description: string - } - vlan: [string]: { - id: int - description: string - } - host: [string]: { - device: string - description: string - } - ... -} - -#UserData: { - autnums: [string]: { - asn: int - name: string - country: string - } -} - #Config: { clickhouseAdminPassword: string clickhouseReadonlyPassword: string enableClickhouseIngress: bool - sampler: #SamplerConfig - userData: #UserData + sampler: _ + userData: _ } ClickHouseInstallation: netmeta: spec: { @@ -78,7 +50,10 @@ ClickHouseInstallation: netmeta: spec: { "readonly/readonly": "1" "readonly/constraints/additional_table_filters/changeable_in_readonly": "" } - files: [string]: string + files: (chBase & #Config & {#Config: { + dataPath: configuration.settings.format_schema_path + risinfoURL: "http://risinfo" + }}).files } templates: { volumeClaimTemplates: [{ diff --git a/deploy/single-node/schema/schema.cue b/deploy/single-node/schema/schema.cue index 45d0389..f53fb7b 100644 --- a/deploy/single-node/schema/schema.cue +++ b/deploy/single-node/schema/schema.cue @@ -7,6 +7,7 @@ import reconciler "github.com/monogon-dev/NetMeta/reconciler:main" ... } sampler: [DEVICE=string]: isIncomingFlow: [...{[COLUMN=string]: string | int}] + kafkaBrokerList: string } function: [NAME=string]: reconciler.#Function & { diff --git a/deploy/single-node/schema/tables.cue b/deploy/single-node/schema/tables.cue index 4510008..fb5ead1 100644 --- a/deploy/single-node/schema/tables.cue +++ b/deploy/single-node/schema/tables.cue @@ -3,7 +3,7 @@ package schema table: flows_queue: { schema: "FlowMessage.proto:FlowMessage" engine: "Kafka" - settings: kafka_broker_list: "netmeta-kafka-bootstrap:9092" + settings: kafka_broker_list: string | *#Config.kafkaBrokerList settings: kafka_topic_list: "flow-messages" settings: kafka_group_name: "clickhouse" settings: kafka_format: "Protobuf" @@ -15,7 +15,7 @@ if #Config.fastNetMon != _|_ { table: fastnetmon_queue: { schema: "traffic_data.proto:TrafficData" engine: "Kafka" - settings: kafka_broker_list: "netmeta-kafka-bootstrap:9092" + settings: kafka_broker_list: string | *#Config.kafkaBrokerList settings: kafka_topic_list: "fastnetmon" settings: kafka_group_name: "clickhouse" settings: kafka_format: "ProtobufSingle"