Skip to content

Commit

Permalink
handle LF/CRLF line endings for all input files on all systems. minor…
Browse files Browse the repository at this point in the history
… fixes
  • Loading branch information
PanAeon committed Nov 28, 2019
1 parent 081c1b5 commit 49437a4
Show file tree
Hide file tree
Showing 9 changed files with 260 additions and 30 deletions.
2 changes: 1 addition & 1 deletion src/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { renderSnap, parseSnap } from './snapshot/parsing'
import { AnnotatedLine, IToken } from './snapshot/model';

import { inspect } from 'util';
import * as diff from 'diff'; // ok, diff doesn't really work. what about text diff
import * as diff from 'diff';

let packageJson = require('../../package.json');

Expand Down
13 changes: 6 additions & 7 deletions src/snapshot/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,27 @@

import * as tm from 'vscode-textmate';
import { AnnotatedLine } from './model';
import { EOL } from 'os';




export async function getVSCodeTokens(registry: tm.Registry, scope: string, source: string): Promise<AnnotatedLine[]> {
return registry.loadGrammar(scope).then((grammar: tm.IGrammar|null) => {

if (!grammar) throw new Error(`Could not load scope ${scope}`);
if (!grammar) {
throw new Error(`Could not load scope ${scope}`);
}

let ruleStack: tm.StackElement = <any>null;

return source.split(EOL).map((line: string, n: number) => {
return source.split(/\r\n|\n/).map((line: string, n: number) => {
var {tokens, ruleStack:ruleStack1} = grammar.tokenizeLine(line, ruleStack);
ruleStack = ruleStack1;

return <AnnotatedLine> {
src: line,
tokens:tokens
};
};
});

});

}
Expand Down
6 changes: 3 additions & 3 deletions src/snapshot/parsing.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@

import {AnnotatedLine, IToken} from "./model"
import {EOL} from 'os'


export function parseSnap(s:string): AnnotatedLine[] {
let result: AnnotatedLine[] = []
let ls = s.split(EOL)
let ls = s.split(/\r\n|\n/)
let i = 0
while(i < ls.length) {
let l = ls[i];
Expand All @@ -31,7 +31,7 @@ export function parseSnap(s:string): AnnotatedLine[] {
i++;
}
}

return result;
}

Expand Down
32 changes: 17 additions & 15 deletions src/unit/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ export {parseGrammarTestCase, GrammarTestCase, TestFailure, missingScopes_}
export async function runGrammarTestCase(registry: tm.Registry, testCase: GrammarTestCase): Promise<TestFailure[]> {
return registry.loadGrammar(testCase.metadata.scope).then((grammar: tm.IGrammar|null) => {

if (!grammar) throw new Error(`Could not load scope ${testCase.metadata.scope}`);
if (!grammar) {
throw new Error(`Could not load scope ${testCase.metadata.scope}`);
}

const assertions = toMap((x) => x.sourceLineNumber, testCase.assertions)

Expand All @@ -34,28 +36,28 @@ export async function runGrammarTestCase(registry: tm.Registry, testCase: Gramma
failures.push(<TestFailure>{
missing: requiredScopes,
unexpected: [],
actual: [],
line: testCaseLineNumber,
srcLine: n,
start: from,
actual: [],
line: testCaseLineNumber,
srcLine: n,
start: from,
end: to
});
} else {
xs.forEach((token) => {
const unexpected = excludedScopes.filter((s) => { return token.scopes.includes(s) })
const missing = missingScopes_(requiredScopes, token.scopes)
const missing = missingScopes_(requiredScopes, token.scopes)

if (missing.length || unexpected.length) {
failures.push(<TestFailure>{
missing: missing,
failures.push(<TestFailure>{
missing: missing,
actual: token.scopes,
unexpected: unexpected,
line: testCaseLineNumber,
srcLine: n,
start: token.startIndex,
end: token.endIndex
line: testCaseLineNumber,
srcLine: n,
start: token.startIndex,
end: token.endIndex
});
}
}
});
}
})
Expand Down Expand Up @@ -105,7 +107,7 @@ function missingScopes_(rs:string[], as: string[]):string[] {
i++
}
}

return j === rs.length ? [] : rs.slice(j);
}

Expand Down
7 changes: 3 additions & 4 deletions src/unit/parsing.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@

import { ScopeAssertion, TestCaseMetadata, LineAssertion, GrammarTestCase } from './model';
import { EOL } from 'os';

const leftArrowAssertRegex = /^\s*<([~]*)([-]+)((?:\s*\w[-\w.]*)*)(?:\s*-)?((?:\s*\w[-\w.]*)*)\s*$/
const upArrowAssertRegex = /^\s*(\^+)((?:\s*\w[-\w.]*)*)(?:\s*-)?((?:\s*\w[-\w.]*)*)\s*$/
Expand Down Expand Up @@ -53,7 +52,7 @@ let headerErrorMessage = "Expecting the first line in the syntax test file to be

let headerRegex = /^([^\s]+)\s+SYNTAX\s+TEST\s+"([^"]+)"(?:\s+\"([^"]+)\")?\s*$/

/*
/*
parse the first line with the format:
<comment character(s)> SYNTAX TEST "<language identifier>" <"description">?
*/
Expand All @@ -67,7 +66,7 @@ export function parseHeader(as: string[]): TestCaseMetadata {
throw new Error(headerErrorMessage);
} else {
let [_, commentToken, scope, description = ""] = matchResult;

return <TestCaseMetadata>{
commentToken: commentToken,
scope: scope,
Expand All @@ -79,7 +78,7 @@ export function parseHeader(as: string[]): TestCaseMetadata {

export function parseGrammarTestCase(str: string): GrammarTestCase {
let headerLength = 1;
let lines = str.split(EOL);
let lines = str.split(/\r\n|\n/);
let metadata = parseHeader(lines)
let { commentToken } = metadata
let rest = lines.slice(headerLength)
Expand Down
1 change: 1 addition & 0 deletions test/resources/aspnet.razor
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
@@
2 changes: 2 additions & 0 deletions test/resources/aspnet.razor.snap
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
>@@
#^^ text.aspnetcorerazor
219 changes: 219 additions & 0 deletions test/resources/aspnetcorerazor.tmLanguage.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
{
"name": "ASP.NET Razor",
"scopeName": "text.aspnetcorerazor",
"patterns": [
{
"include": "#explicit-razor-expression"
},
{
"include": "#escaped-transition"
},
{
"include": "#implicit-expression"
},
{
"include": "text.html.basic"
}
],
"repository": {
"escaped-transition": {
"match": "@@"
},
"transition": {
"match": "@",
"name": "keyword.control.razor.transition"
},
"explicit-razor-expression": {
"name": "meta.expression.explicit.cshtml",
"begin": "(@)\\(",
"captures": {
"0": {
"name": "keyword.control.cshtml"
}
},
"patterns": [
{
"include": "source.cs"
}
],
"end": "\\)"
},
"implicit-expression": {
"begin": "(@)",
"name": "razor.test.implicitExpression",
"beginCaptures": {
"1": {
"patterns": [
{
"include": "#transition"
}
]
}
},
"patterns": [
{
"include": "#implicit-expression-invocation-start"
},
{
"include": "#implicit-expression-accessor-start"
}
],
"end": "(?=[\\s<])"
},
"implicit-expression-invocation-start": {
"begin": "([_[:alpha:]][_[:alnum:]]*)(?=\\()",
"beginCaptures": {
"1": {
"name": "entity.name.function.cs"
}
},
"patterns": [
{
"include": "#implicit-expression-body"
}
],
"end": "(?=[\\s<])"
},
"implicit-expression-accessor-start": {
"begin": "([_[:alpha:]][_[:alnum:]]*)",
"beginCaptures": {
"1": {
"name": "variable.other.object.cs"
}
},
"patterns": [
{
"include": "#implicit-expression-body"
}
],
"end": "(?=[\\s<])"
},
"implicit-expression-body": {
"name": "razor.test.implicitExpressionBody",
"patterns": [
{
"include": "#balanced-parenthesis-csharp"
},
{
"include": "#balanced-brackets-csharp"
},
{
"include": "#implicit-expression-invocation"
},
{
"include": "#implicit-expression-accessor"
},
{
"include": "#implicit-expression-extension"
}
],
"end": "(?=[\\s<])"
},
"implicit-expression-accessor": {
"match": "(?<=\\.)[_[:alpha:]][_[:alnum:]]*",
"name": "variable.other.object.property.cs"
},
"implicit-expression-invocation": {
"match": "(?<=\\.)[_[:alpha:]][_[:alnum:]]*(?=\\()",
"name": "entity.name.function.cs"
},
"balanced-parenthesis-csharp": {
"begin": "(\\()",
"beginCaptures": {
"1": {
"name": "punctuation.parenthesis.open.cs"
}
},
"name": "razor.test.balanced.parenthesis",
"patterns": [
{
"include": "source.cs"
}
],
"end": "(\\))",
"endCaptures": {
"1": {
"name": "punctuation.parenthesis.close.cs"
}
}
},
"balanced-brackets-csharp": {
"begin": "(\\[)",
"beginCaptures": {
"1": {
"name": "punctuation.squarebracket.open.cs"
}
},
"name": "razor.test.balanced.brackets",
"patterns": [
{
"include": "source.cs"
}
],
"end": "(\\])",
"endCaptures": {
"1": {
"name": "punctuation.squarebracket.close.cs"
}
}
},
"balanced-curlybraces-csharp": {
"begin": "(\\{)",
"beginCaptures": {
"1": {
"name": "punctuation.curlybrace.open.cs"
}
},
"name": "razor.test.balanced.curlybraces",
"patterns": [
{
"include": "source.cs"
}
],
"end": "(\\})",
"endCaptures": {
"1": {
"name": "punctuation.curlybrace.close.cs"
}
}
},
"implicit-expression-extension": {
"name": "razor.test.implicitExpression.Extensions",
"patterns": [
{
"include": "#implicit-expression-dot-extension"
},
{
"include": "#implicit-expression-null-conditional-extension"
},
{
"include": "#implicit-expression-null-forgiveness-extension"
}
]
},
"implicit-expression-dot-extension": {
"match": "(\\.)(?=[_[:alpha:]][_[:alnum:]]*)",
"captures": {
"1": {
"name": "punctuation.accessor.cs"
}
}
},
"implicit-expression-null-conditional-extension": {
"match": "(\\?)(?=[.\\[])",
"captures": {
"1": {
"name": "keyword.operator.null-conditional.cs"
}
}
},
"implicit-expression-null-forgiveness-extension": {
"match": "(\\!)(?=(\\.[_[:alpha:]][_[:alnum:]]*)|\\?|[\\[\\(])",
"captures": {
"1": {
"name": "keyword.operator.logical.cs"
}
}
}
}
}
Loading

0 comments on commit 49437a4

Please sign in to comment.