From b04abc09f02b060a673f0941cb0f43db44b7d1f2 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Mon, 23 Sep 2024 16:41:34 +0200 Subject: [PATCH 1/8] Rust: extract syntax errors --- rust/extractor/src/main.rs | 5 ++- rust/extractor/src/translate/base.rs | 18 +++++++++-- rust/extractor/src/trap.rs | 46 +++++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index 0300e3c1f3b0..2fd672765570 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -27,15 +27,14 @@ fn extract( let parse = ra_ap_syntax::ast::SourceFile::parse(&input, Edition::CURRENT); for err in parse.errors() { - let (start, _) = translator.location(err.range()); - log::warn!("{}:{}:{}: {}", display_path, start.line, start.col, err); + translator.emit_parse_error(display_path.as_ref(), err); } if let Some(ast) = SourceFile::cast(parse.syntax_node()) { translator.emit_source_file(ast); - translator.trap.commit()? } else { log::warn!("Skipped {}", display_path); } + translator.trap.commit()?; Ok(()) } fn main() -> anyhow::Result<()> { diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index b0d42b1dbee3..6d261daf80fe 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -1,10 +1,9 @@ -use crate::trap::TrapFile; +use crate::trap::{DiagnosticSeverity, TrapFile}; use crate::trap::{Label, TrapClass}; use codeql_extractor::trap::{self}; use ra_ap_ide_db::line_index::{LineCol, LineIndex}; use ra_ap_syntax::ast::RangeItem; -use ra_ap_syntax::TextSize; -use ra_ap_syntax::{ast, TextRange}; +use ra_ap_syntax::{ast, SyntaxError, TextRange, TextSize}; pub trait TextValue { fn try_get_text(&self) -> Option; } @@ -83,4 +82,17 @@ impl Translator { let (start, end) = self.location(node.syntax().text_range()); self.trap.emit_location(self.label, label, start, end) } + pub fn emit_parse_error(&mut self, path: &str, err: SyntaxError) { + let (start, end) = self.location(err.range()); + log::warn!("{}:{}:{}: {}", path, start.line, start.col, err); + let message = err.to_string(); + let location = self.trap.emit_location_label(self.label, start, end); + self.trap.emit_diagnostic( + DiagnosticSeverity::Warning, + "parse_error".to_owned(), + message.clone(), + message, + location, + ); + } } diff --git a/rust/extractor/src/trap.rs b/rust/extractor/src/trap.rs index 6ad1742557dc..0d549ea4dc7a 100644 --- a/rust/extractor/src/trap.rs +++ b/rust/extractor/src/trap.rs @@ -128,19 +128,25 @@ pub struct TrapFile { compression: Compression, } +#[derive(Copy, Clone)] +pub enum DiagnosticSeverity { + Debug = 10, + Info = 20, + Warning = 30, + Error = 40, +} impl TrapFile { - pub fn emit_location( + pub fn emit_location_label( &mut self, file_label: UntypedLabel, - entity_label: Label, start: LineCol, end: LineCol, - ) { + ) -> UntypedLabel { let start_line = 1 + start.line as usize; let start_column = 1 + start.col as usize; let end_line = 1 + end.line as usize; let end_column = 1 + end.col as usize; - let location_label = extractor::location_label( + extractor::location_label( &mut self.writer, trap::Location { file_label, @@ -149,13 +155,43 @@ impl TrapFile { end_line, end_column, }, - ); + ) + } + pub fn emit_location( + &mut self, + file_label: UntypedLabel, + entity_label: Label, + start: LineCol, + end: LineCol, + ) { + let location_label = self.emit_location_label(file_label, start, end); self.writer.add_tuple( "locatable_locations", vec![entity_label.into(), location_label.into()], ); } + pub fn emit_diagnostic( + &mut self, + severity: DiagnosticSeverity, + error_tag: String, + error_message: String, + full_error_message: String, + location: UntypedLabel, + ) { + let label = self.writer.fresh_id(); + self.writer.add_tuple( + "diagnostics", + vec![ + trap::Arg::Label(label), + trap::Arg::Int(severity as usize), + trap::Arg::String(error_tag), + trap::Arg::String(error_message), + trap::Arg::String(full_error_message), + trap::Arg::Label(location), + ], + ); + } pub fn emit_file(&mut self, absolute_path: &Path) -> trap::Label { extractor::populate_file(&mut self.writer, absolute_path) } From bdb4d89f9f32d08bdb349933e4cf4a85dafd39a4 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Mon, 23 Sep 2024 17:15:26 +0200 Subject: [PATCH 2/8] Rust: add Diagnostics.qll --- config/identical-files.json | 4 ++ rust/ql/lib/codeql/rust/Diagnostics.qll | 52 +++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 rust/ql/lib/codeql/rust/Diagnostics.qll diff --git a/config/identical-files.json b/config/identical-files.json index d14052c12d83..89a1e6fc363a 100644 --- a/config/identical-files.json +++ b/config/identical-files.json @@ -355,5 +355,9 @@ "Python model summaries test extension": [ "python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml", "python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml" + ], + "Diagnostics.qll": [ + "ruby/ql/lib/codeql/ruby/Diagnostics.qll", + "rust/ql/lib/codeql/rust/Diagnostics.qll" ] } diff --git a/rust/ql/lib/codeql/rust/Diagnostics.qll b/rust/ql/lib/codeql/rust/Diagnostics.qll new file mode 100644 index 000000000000..faf7b8420a0e --- /dev/null +++ b/rust/ql/lib/codeql/rust/Diagnostics.qll @@ -0,0 +1,52 @@ +private import codeql.Locations + +/** A diagnostic emitted during extraction, such as a parse error */ +class Diagnostic extends @diagnostic { + int severity; + string tag; + string message; + string fullMessage; + Location location; + + Diagnostic() { diagnostics(this, severity, tag, message, fullMessage, location) } + + /** + * Gets the numerical severity level associated with this diagnostic. + */ + int getSeverity() { result = severity } + + /** Gets a string representation of the severity of this diagnostic. */ + string getSeverityText() { + severity = 10 and result = "Debug" + or + severity = 20 and result = "Info" + or + severity = 30 and result = "Warning" + or + severity = 40 and result = "Error" + } + + /** Gets the error code associated with this diagnostic, e.g. parse_error. */ + string getTag() { result = tag } + + /** + * Gets the error message text associated with this diagnostic. + */ + string getMessage() { result = message } + + /** + * Gets the full error message text associated with this diagnostic. + */ + string getFullMessage() { result = fullMessage } + + /** Gets the source location of this diagnostic. */ + Location getLocation() { result = location } + + /** Gets a textual representation of this diagnostic. */ + string toString() { result = this.getMessage() } +} + +/** A diagnostic relating to a particular error in extracting a file. */ +class ExtractionError extends Diagnostic { + ExtractionError() { this.getTag() = "parse_error" } +} From c596205416011bd28bedb750088553654f45d435 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Mon, 23 Sep 2024 17:16:57 +0200 Subject: [PATCH 3/8] Rust: add some more diagnostics queries --- .../queries/diagnostics/ExtractionErrors.ql | 18 ++++++++++++++++++ .../NumberOfFilesExtractedWithErrors.ql | 15 +++++++++++++++ .../NumberOfSuccessfullyExtractedFiles.ql | 15 +++++++++++++++ .../diagnostics/ExtractionErrors.expected | 6 ++++++ .../diagnostics/ExtractionErrors.qlref | 1 + 5 files changed, 55 insertions(+) create mode 100644 rust/ql/src/queries/diagnostics/ExtractionErrors.ql create mode 100644 rust/ql/src/queries/summary/NumberOfFilesExtractedWithErrors.ql create mode 100644 rust/ql/src/queries/summary/NumberOfSuccessfullyExtractedFiles.ql create mode 100644 rust/ql/test/query-tests/diagnostics/ExtractionErrors.expected create mode 100644 rust/ql/test/query-tests/diagnostics/ExtractionErrors.qlref diff --git a/rust/ql/src/queries/diagnostics/ExtractionErrors.ql b/rust/ql/src/queries/diagnostics/ExtractionErrors.ql new file mode 100644 index 000000000000..a04c4e618c41 --- /dev/null +++ b/rust/ql/src/queries/diagnostics/ExtractionErrors.ql @@ -0,0 +1,18 @@ +/** + * @name Extraction errors + * @description List all extraction errors for files in the source code directory. + * @kind diagnostic + * @id rust/diagnostics/extraction-errors + */ + +import codeql.rust.Diagnostics +import codeql.files.FileSystem + +/** Gets the SARIF severity to associate an error. */ +int getSeverity() { result = 2 } + +from ExtractionError error, File f +where + f = error.getLocation().getFile() and + exists(f.getRelativePath()) +select error, "Extraction failed in " + f + " with error " + error.getMessage(), getSeverity() diff --git a/rust/ql/src/queries/summary/NumberOfFilesExtractedWithErrors.ql b/rust/ql/src/queries/summary/NumberOfFilesExtractedWithErrors.ql new file mode 100644 index 000000000000..14427158c369 --- /dev/null +++ b/rust/ql/src/queries/summary/NumberOfFilesExtractedWithErrors.ql @@ -0,0 +1,15 @@ +/** + * @id rust/summary/number-of-files-extracted-with-errors + * @name Total number of Rust files that were extracted with errors + * @description The total number of Rust code files that we extracted, but where + * at least one extraction error occurred in the process. + * @kind metric + * @tags summary + */ + +import codeql.files.FileSystem +import codeql.rust.Diagnostics + +select count(File f | + exists(ExtractionError e | e.getLocation().getFile() = f) and exists(f.getRelativePath()) + ) diff --git a/rust/ql/src/queries/summary/NumberOfSuccessfullyExtractedFiles.ql b/rust/ql/src/queries/summary/NumberOfSuccessfullyExtractedFiles.ql new file mode 100644 index 000000000000..b8a64f60512d --- /dev/null +++ b/rust/ql/src/queries/summary/NumberOfSuccessfullyExtractedFiles.ql @@ -0,0 +1,15 @@ +/** + * @id rust/summary/number-of-successfully-extracted-files + * @name Total number of Rust files that were extracted without error + * @description The total number of Rust code files that we extracted without + * encountering any extraction errors + * @kind metric + * @tags summary + */ + +import codeql.rust.Diagnostics +import codeql.files.FileSystem + +select count(File f | + not exists(ExtractionError e | e.getLocation().getFile() = f) and exists(f.getRelativePath()) + ) diff --git a/rust/ql/test/query-tests/diagnostics/ExtractionErrors.expected b/rust/ql/test/query-tests/diagnostics/ExtractionErrors.expected new file mode 100644 index 000000000000..b6aaf7b6d373 --- /dev/null +++ b/rust/ql/test/query-tests/diagnostics/ExtractionErrors.expected @@ -0,0 +1,6 @@ +| does_not_compile.rs:2:6:2:5 | expected SEMICOLON | Extraction failed in does_not_compile.rs with error expected SEMICOLON | 2 | +| does_not_compile.rs:2:9:2:8 | expected SEMICOLON | Extraction failed in does_not_compile.rs with error expected SEMICOLON | 2 | +| does_not_compile.rs:2:13:2:12 | expected SEMICOLON | Extraction failed in does_not_compile.rs with error expected SEMICOLON | 2 | +| does_not_compile.rs:2:21:2:20 | expected SEMICOLON | Extraction failed in does_not_compile.rs with error expected SEMICOLON | 2 | +| does_not_compile.rs:2:26:2:25 | expected SEMICOLON | Extraction failed in does_not_compile.rs with error expected SEMICOLON | 2 | +| does_not_compile.rs:2:32:2:31 | expected field name or number | Extraction failed in does_not_compile.rs with error expected field name or number | 2 | diff --git a/rust/ql/test/query-tests/diagnostics/ExtractionErrors.qlref b/rust/ql/test/query-tests/diagnostics/ExtractionErrors.qlref new file mode 100644 index 000000000000..ffbdb0a7b1b5 --- /dev/null +++ b/rust/ql/test/query-tests/diagnostics/ExtractionErrors.qlref @@ -0,0 +1 @@ +queries/diagnostics/ExtractionErrors.ql \ No newline at end of file From 9b8cbdad49910f329b857743c5c83574a4ff899e Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Mon, 23 Sep 2024 17:43:23 +0200 Subject: [PATCH 4/8] Add QLDoc to Diagnostics.qll --- ruby/ql/lib/codeql/ruby/Diagnostics.qll | 2 ++ rust/ql/lib/codeql/rust/Diagnostics.qll | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ruby/ql/lib/codeql/ruby/Diagnostics.qll b/ruby/ql/lib/codeql/ruby/Diagnostics.qll index faf7b8420a0e..5902bb594141 100644 --- a/ruby/ql/lib/codeql/ruby/Diagnostics.qll +++ b/ruby/ql/lib/codeql/ruby/Diagnostics.qll @@ -1,3 +1,5 @@ +/** Provides classes relating to extraction diagnostics. */ + private import codeql.Locations /** A diagnostic emitted during extraction, such as a parse error */ diff --git a/rust/ql/lib/codeql/rust/Diagnostics.qll b/rust/ql/lib/codeql/rust/Diagnostics.qll index faf7b8420a0e..5902bb594141 100644 --- a/rust/ql/lib/codeql/rust/Diagnostics.qll +++ b/rust/ql/lib/codeql/rust/Diagnostics.qll @@ -1,3 +1,5 @@ +/** Provides classes relating to extraction diagnostics. */ + private import codeql.Locations /** A diagnostic emitted during extraction, such as a parse error */ From ca2b8ef6c0db5c0b25e086a01b8c23786fdf8e7e Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Tue, 24 Sep 2024 10:02:25 +0200 Subject: [PATCH 5/8] Apply suggestions from code review Co-authored-by: Geoffrey White <40627776+geoffw0@users.noreply.github.com> --- .../src/queries/summary/NumberOfFilesExtractedWithErrors.ql | 4 ++-- .../src/queries/summary/NumberOfSuccessfullyExtractedFiles.ql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/ql/src/queries/summary/NumberOfFilesExtractedWithErrors.ql b/rust/ql/src/queries/summary/NumberOfFilesExtractedWithErrors.ql index 14427158c369..1af3f0f88ec0 100644 --- a/rust/ql/src/queries/summary/NumberOfFilesExtractedWithErrors.ql +++ b/rust/ql/src/queries/summary/NumberOfFilesExtractedWithErrors.ql @@ -1,8 +1,8 @@ /** * @id rust/summary/number-of-files-extracted-with-errors * @name Total number of Rust files that were extracted with errors - * @description The total number of Rust code files that we extracted, but where - * at least one extraction error occurred in the process. + * @description The total number of Rust files in the source code directory that + * were extracted, but where at least one extraction error occurred in the process. * @kind metric * @tags summary */ diff --git a/rust/ql/src/queries/summary/NumberOfSuccessfullyExtractedFiles.ql b/rust/ql/src/queries/summary/NumberOfSuccessfullyExtractedFiles.ql index b8a64f60512d..eb86577b4b80 100644 --- a/rust/ql/src/queries/summary/NumberOfSuccessfullyExtractedFiles.ql +++ b/rust/ql/src/queries/summary/NumberOfSuccessfullyExtractedFiles.ql @@ -1,8 +1,8 @@ /** * @id rust/summary/number-of-successfully-extracted-files * @name Total number of Rust files that were extracted without error - * @description The total number of Rust code files that we extracted without - * encountering any extraction errors + * @description The total number of Rust files in the source code directory that + * were extracted without encountering any extraction errors. * @kind metric * @tags summary */ From 37f264df74f41c624b970b1827724f508c03dc19 Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Tue, 24 Sep 2024 11:11:48 +0200 Subject: [PATCH 6/8] Rust: correct line and column numbers in printed error messages --- rust/extractor/src/translate/base.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index 6d261daf80fe..9bca31421642 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -84,7 +84,7 @@ impl Translator { } pub fn emit_parse_error(&mut self, path: &str, err: SyntaxError) { let (start, end) = self.location(err.range()); - log::warn!("{}:{}:{}: {}", path, start.line, start.col, err); + log::warn!("{}:{}:{}: {}", path, start.line + 1, start.col + 1, err); let message = err.to_string(); let location = self.trap.emit_location_label(self.label, start, end); self.trap.emit_diagnostic( From 1953e4f9719353131eebdc0bd6687f91bea6dd1d Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Tue, 24 Sep 2024 11:12:40 +0200 Subject: [PATCH 7/8] Rust: handle UTF-8 decoding errors --- rust/extractor/src/main.rs | 43 +++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index 2fd672765570..81032636ceb0 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -1,13 +1,48 @@ use anyhow::Context; use ra_ap_ide_db::line_index::LineIndex; use ra_ap_parser::Edition; +use std::borrow::Cow; mod archive; mod config; pub mod generated; mod translate; pub mod trap; use ra_ap_syntax::ast::SourceFile; -use ra_ap_syntax::AstNode; +use ra_ap_syntax::{AstNode, SyntaxError, TextRange, TextSize}; + +fn from_utf8_lossy(v: &[u8]) -> (Cow<'_, str>, Option) { + let mut iter = v.utf8_chunks(); + let (first_valid, first_invalid) = if let Some(chunk) = iter.next() { + let valid = chunk.valid(); + let invalid = chunk.invalid(); + if invalid.is_empty() { + debug_assert_eq!(valid.len(), v.len()); + return (Cow::Borrowed(valid), None); + } + (valid, invalid) + } else { + return (Cow::Borrowed(""), None); + }; + + const REPLACEMENT: &str = "\u{FFFD}"; + let error_start = first_valid.len() as u32; + let error_end = error_start + first_invalid.len() as u32; + let error_range = TextRange::new(TextSize::new(error_start), TextSize::new(error_end)); + let error = SyntaxError::new("invalid utf-8 sequence".to_owned(), error_range); + let mut res = String::with_capacity(v.len()); + res.push_str(first_valid); + + res.push_str(REPLACEMENT); + + for chunk in iter { + res.push_str(chunk.valid()); + if !chunk.invalid().is_empty() { + res.push_str(REPLACEMENT); + } + } + + (Cow::Owned(res), Some(error)) +} fn extract( archiver: &archive::Archiver, @@ -18,13 +53,15 @@ fn extract( let file = std::fs::canonicalize(&file).unwrap_or(file); archiver.archive(&file); let input = std::fs::read(&file)?; - let input = String::from_utf8(input)?; + let (input, err) = from_utf8_lossy(&input); let line_index = LineIndex::new(&input); let display_path = file.to_string_lossy(); let mut trap = traps.create("source", &file); let label = trap.emit_file(&file); let mut translator = translate::Translator::new(trap, label, line_index); - + if let Some(err) = err { + translator.emit_parse_error(display_path.as_ref(), err); + } let parse = ra_ap_syntax::ast::SourceFile::parse(&input, Edition::CURRENT); for err in parse.errors() { translator.emit_parse_error(display_path.as_ref(), err); From 57148110715e672c5823e84390fbb8577d2ed5bc Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Tue, 24 Sep 2024 16:20:50 +0200 Subject: [PATCH 8/8] Rust: fix panic when the last character in a range is multi-byte --- rust/extractor/src/translate/base.rs | 23 +++++++---- .../ql/test/extractor-tests/utf8/ast.expected | 39 +++++++++++++++++++ rust/ql/test/extractor-tests/utf8/ast.ql | 3 ++ .../extractor-tests/utf8/utf8-identifiers.rs | 12 ++++++ 4 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 rust/ql/test/extractor-tests/utf8/ast.expected create mode 100644 rust/ql/test/extractor-tests/utf8/ast.ql create mode 100644 rust/ql/test/extractor-tests/utf8/utf8-identifiers.rs diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index 9bca31421642..063a59d8937e 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -3,7 +3,7 @@ use crate::trap::{Label, TrapClass}; use codeql_extractor::trap::{self}; use ra_ap_ide_db::line_index::{LineCol, LineIndex}; use ra_ap_syntax::ast::RangeItem; -use ra_ap_syntax::{ast, SyntaxError, TextRange, TextSize}; +use ra_ap_syntax::{ast, SyntaxError, TextRange}; pub trait TextValue { fn try_get_text(&self) -> Option; } @@ -70,12 +70,21 @@ impl Translator { } pub fn location(&self, range: TextRange) -> (LineCol, LineCol) { let start = self.line_index.line_col(range.start()); - let end = self.line_index.line_col( - range - .end() - .checked_sub(TextSize::new(1)) - .unwrap_or(range.end()), - ); + let range_end = range.end(); + // QL end positions are inclusive, while TextRange offsets are exclusive and point at the position + // right after the last character of the range. We need to shift the end offset one character to the left to + // get the right inclusive QL position. Unfortunately, simply subtracting `1` from the end-offset may cause + // the offset to point in the middle of a mult-byte character, resulting in a `panic`. Therefore we use `try_line_col` + // with decreasing offsets to find the start of the last character included in the range. + for i in 1..4 { + if let Some(end) = range_end + .checked_sub(i.into()) + .and_then(|x| self.line_index.try_line_col(x)) + { + return (start, end); + } + } + let end = self.line_index.line_col(range_end); (start, end) } pub fn emit_location(&mut self, label: Label, node: impl ast::AstNode) { diff --git a/rust/ql/test/extractor-tests/utf8/ast.expected b/rust/ql/test/extractor-tests/utf8/ast.expected new file mode 100644 index 000000000000..bf1fd078af80 --- /dev/null +++ b/rust/ql/test/extractor-tests/utf8/ast.expected @@ -0,0 +1,39 @@ +| lib.rs:1:1:3:22 | SourceFile | +| lib.rs:2:1:2:8 | Module | +| lib.rs:2:5:2:7 | Name | +| lib.rs:3:1:3:8 | Module | +| lib.rs:3:5:3:8 | Name | +| lib.rs:3:10:3:20 | NameRef | +| lib.rs:3:10:3:20 | Path | +| lib.rs:3:10:3:20 | PathSegment | +| lib.rs:3:10:3:21 | MacroCall | +| utf8-identifiers.rs:1:1:4:6 | foo | +| utf8-identifiers.rs:1:1:12:2 | SourceFile | +| utf8-identifiers.rs:1:4:1:6 | Name | +| utf8-identifiers.rs:1:7:4:1 | GenericParamList | +| utf8-identifiers.rs:2:5:2:6 | Lifetime | +| utf8-identifiers.rs:2:5:2:6 | LifetimeParam | +| utf8-identifiers.rs:3:5:3:5 | Name | +| utf8-identifiers.rs:3:5:3:5 | TypeParam | +| utf8-identifiers.rs:4:2:4:3 | ParamList | +| utf8-identifiers.rs:4:5:4:6 | BlockExpr | +| utf8-identifiers.rs:4:5:4:6 | StmtList | +| utf8-identifiers.rs:6:1:8:1 | Struct | +| utf8-identifiers.rs:6:8:6:8 | Name | +| utf8-identifiers.rs:6:10:8:1 | RecordFieldList | +| utf8-identifiers.rs:7:5:7:5 | Name | +| utf8-identifiers.rs:7:5:7:13 | RecordField | +| utf8-identifiers.rs:7:9:7:13 | NameRef | +| utf8-identifiers.rs:7:9:7:13 | Path | +| utf8-identifiers.rs:7:9:7:13 | PathSegment | +| utf8-identifiers.rs:7:9:7:13 | PathType | +| utf8-identifiers.rs:10:1:10:3 | Visibility | +| utf8-identifiers.rs:10:1:12:1 | main | +| utf8-identifiers.rs:10:8:10:11 | Name | +| utf8-identifiers.rs:10:12:10:13 | ParamList | +| utf8-identifiers.rs:10:15:12:1 | BlockExpr | +| utf8-identifiers.rs:10:15:12:1 | StmtList | +| utf8-identifiers.rs:11:5:11:24 | LetStmt | +| utf8-identifiers.rs:11:9:11:9 | IdentPat | +| utf8-identifiers.rs:11:9:11:9 | Name | +| utf8-identifiers.rs:11:14:11:23 | LiteralExpr | diff --git a/rust/ql/test/extractor-tests/utf8/ast.ql b/rust/ql/test/extractor-tests/utf8/ast.ql new file mode 100644 index 000000000000..420f7e94eb9a --- /dev/null +++ b/rust/ql/test/extractor-tests/utf8/ast.ql @@ -0,0 +1,3 @@ +import codeql.rust.elements + +select any(AstNode n) diff --git a/rust/ql/test/extractor-tests/utf8/utf8-identifiers.rs b/rust/ql/test/extractor-tests/utf8/utf8-identifiers.rs new file mode 100644 index 000000000000..579dc82ab5f0 --- /dev/null +++ b/rust/ql/test/extractor-tests/utf8/utf8-identifiers.rs @@ -0,0 +1,12 @@ +fn foo< + 'β, + γ +>() {} + +struct X { + δ: usize +} + +pub fn main() { + let α = 0.00001f64; +}