diff --git a/pom.xml b/pom.xml index c9b7c93..7a41fc0 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.jruby jruby-prism jar - 1.5.0 + 2.0.0-SNAPSHOT jruby-prism Java portion of JRuby Prism parser support. @@ -66,12 +66,12 @@ org.jruby jruby-base - 10.0.0.0-SNAPSHOT + 10.0.2.0 - com.prism - java-prism - 999-SNAPSHOT + org.jruby + chicory-prism + 0.0.1-SNAPSHOT @@ -116,45 +116,6 @@ - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - sign-artifacts - verify - - sign - - - - - - maven-source-plugin - - - attach-sources - - jar-no-fork - - - - - - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - - none - - @@ -210,5 +171,56 @@ + + release + + + + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + --pinentry-mode + loopback + + + + + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + none + + + + + diff --git a/src/main/java/org/jruby/prism/ParserProviderPrism.java b/src/main/java/org/jruby/prism/ParserProviderPrism.java index e81cd3d..07beb81 100644 --- a/src/main/java/org/jruby/prism/ParserProviderPrism.java +++ b/src/main/java/org/jruby/prism/ParserProviderPrism.java @@ -1,31 +1,43 @@ package org.jruby.prism; +import jnr.ffi.LibraryLoader; import org.jruby.Ruby; import org.jruby.ir.builder.IRBuilderFactory; import org.jruby.parser.Parser; +import org.jruby.parser.ParserManager; import org.jruby.parser.ParserProvider; -import org.jruby.prism.parser.ParserPrism; -import org.jruby.prism.parser.ParserBindingPrism; import org.jruby.prism.builder.IRBuilderFactoryPrism; +import org.jruby.prism.parser.ParserBindingPrism; +import org.jruby.prism.parser.ParserPrismNative; +import org.jruby.prism.parser.ParserPrismWasm; -import jnr.ffi.LibraryLoader; +import java.io.File; public class ParserProviderPrism implements ParserProvider { private static ParserBindingPrism prismLibrary; public void initialize(String path) { - if (prismLibrary != null) { - System.out.println("Prism already initialized"); - return; + if (new File(path).exists()) { + if (prismLibrary != null) { + System.out.println("Prism already initialized"); + return; + } + prismLibrary = LibraryLoader.create(ParserBindingPrism.class).load(path); + // We do something extra here as a side-effect which is how we get an UnsatisfiedLinkError + // If the library didn't in fact find the .so or has other loading problems. + ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary)); + } else { + prismLibrary = null; } - prismLibrary = LibraryLoader.create(ParserBindingPrism.class).load(path); - // We do something extra here as a side-effect which is how we get an UnsatisfiedLinkError - // If the library didn't in fact find the .so or has other loading problems. - ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary)); } public Parser getParser(Ruby runtime) { - return new ParserPrism(runtime, prismLibrary); + if (ParserManager.PARSER_WASM || prismLibrary == null) { + // uninitialized dynamic lib or wasm requested + return new ParserPrismWasm(runtime); + } + + return new ParserPrismNative(runtime, prismLibrary); } public IRBuilderFactory getBuilderFactory() { diff --git a/src/main/java/org/jruby/prism/parser/ParserPrism.java b/src/main/java/org/jruby/prism/parser/ParserPrismBase.java similarity index 72% rename from src/main/java/org/jruby/prism/parser/ParserPrism.java rename to src/main/java/org/jruby/prism/parser/ParserPrismBase.java index f11dee3..e4e28b5 100644 --- a/src/main/java/org/jruby/prism/parser/ParserPrism.java +++ b/src/main/java/org/jruby/prism/parser/ParserPrismBase.java @@ -11,7 +11,6 @@ import org.jruby.ext.coverage.CoverageData; import org.jruby.management.ParserStats; import org.jruby.parser.Parser; -import org.jruby.parser.ParserManager; import org.jruby.parser.ParserType; import org.jruby.parser.StaticScope; import org.jruby.runtime.DynamicScope; @@ -21,10 +20,16 @@ import org.jruby.util.ByteList; import org.jruby.util.CommonByteLists; import org.jruby.util.io.ChannelHelper; -import org.prism.Nodes; -import org.prism.Nodes.*; +import org.prism.Nodes.ArgumentsNode; +import org.prism.Nodes.CallNode; +import org.prism.Nodes.CallNodeFlags; +import org.prism.Nodes.GlobalVariableReadNode; +import org.prism.Nodes.GlobalVariableWriteNode; +import org.prism.Nodes.Node; +import org.prism.Nodes.ProgramNode; +import org.prism.Nodes.StatementsNode; +import org.prism.Nodes.WhileNode; import org.prism.ParsingOptions; -import org.prism.Prism; import java.io.ByteArrayInputStream; import java.io.DataInputStream; @@ -32,28 +37,29 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; +import java.util.stream.IntStream; import static org.jruby.api.Convert.asSymbol; import static org.jruby.lexer.LexingCommon.DOLLAR_UNDERSCORE; import static org.jruby.parser.ParserType.EVAL; import static org.jruby.parser.ParserType.MAIN; -public class ParserPrism extends Parser { - private boolean parserTiming = org.jruby.util.cli.Options.PARSER_SUMMARY.load(); +public abstract class ParserPrismBase extends Parser { + protected boolean parserTiming = org.jruby.util.cli.Options.PARSER_SUMMARY.load(); - private final ParserBindingPrism prismLibrary; - - public ParserPrism(Ruby runtime, ParserBindingPrism prismLibrary) { + public ParserPrismBase(Ruby runtime) { super(runtime); - this.prismLibrary = prismLibrary; } + protected abstract byte[] parse(byte[] source, int sourceLength, byte[] metadata); + @Override public ParseResult parse(String fileName, int lineNumber, ByteList content, DynamicScope existingScope, ParserType type) { int sourceLength = content.realSize(); byte[] source = content.begin() == 0 ? content.unsafeBytes() : content.bytes(); - byte[] metadata = generateMetadata(fileName, lineNumber, content.getEncoding(), existingScope, type); + byte[] metadata = generateMetadata(fileName, lineNumber, content.getEncoding(), existingScope); byte[] serialized = parse(source, sourceLength, metadata); return parseInternal(fileName, existingScope, source, serialized, type); } @@ -113,10 +119,10 @@ private ParseResult parseInternal(String fileName, DynamicScope blockScope, byte coverageMode = runtime.getCoverageData().getMode(); } - ParseResultPrism result = new ParseResultPrism(fileName, source, (Nodes.ProgramNode) res.value, res.source, encoding, coverageMode); + ParseResultPrism result = new ParseResultPrism(fileName, source, (ProgramNode) res.value, res.source, encoding, coverageMode); if (blockScope != null) { if (type == MAIN) { // update TOPLEVEL_BINDNG - RubySymbol[] locals = ((Nodes.ProgramNode) result.getAST()).locals; + RubySymbol[] locals = ((ProgramNode) result.getAST()).locals; for (int i = 0; i < locals.length; i++) { blockScope.getStaticScope().addVariableThisScope(locals[i].idString()); } @@ -148,7 +154,7 @@ private void populateScriptData(byte[] source, Encoding encoding, RubyArray line protected ParseResult parse(String fileName, int lineNumber, InputStream in, Encoding encoding, DynamicScope existingScope, ParserType type) { byte[] source = getSourceAsBytes(fileName, in); - byte[] metadata = generateMetadata(fileName, lineNumber, encoding, existingScope, type); + byte[] metadata = generateMetadata(fileName, lineNumber, encoding, existingScope); byte[] serialized = parse(source, source.length, metadata); return parseInternal(fileName, existingScope, source, serialized, type); } @@ -174,92 +180,58 @@ private byte[] loadFully(String fileName, InputStream in) { } } + // lineNumber (0-indexed) + private byte[] generateMetadata(String fileName, int lineNumber, Encoding encoding, DynamicScope scope) { + return ParsingOptions.serialize( + fileName.getBytes(), + lineNumber + 1, + encoding.getName(), + (runtime.getInstanceConfig().isFrozenStringLiteral() instanceof Boolean bool && bool), + commandLineFromConfig(runtime.getInstanceConfig()), + ParsingOptions.SyntaxVersion.V4_0, + false, + true, + false, + evalScopes(scope)); + } - private byte[] parse(byte[] source, int sourceLength, byte[] metadata) { - if (ParserManager.PARSER_WASM) return parseChicory(source, sourceLength, metadata); - - long time = 0; - if (parserTiming) time = System.nanoTime(); - - ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary)); - prismLibrary.pm_buffer_init(buffer); - prismLibrary.pm_serialize_parse(buffer, source, sourceLength, metadata); - if (parserTiming) { - ParserStats stats = runtime.getParserManager().getParserStats(); + private ParsingOptions.Scope[] evalScopes(DynamicScope scope) { + if (scope == null) return new ParsingOptions.Scope[0]; - stats.addPrismTimeCParseSerialize(System.nanoTime() - time); - } + var scopes = new ArrayList(); - int length = buffer.length.intValue(); - byte[] src = new byte[length]; - buffer.value.get().get(0, src, 0, length); + evalScopesRecursive(scope.getStaticScope(), scopes); - return src; + return scopes.toArray(ParsingOptions.Scope[]::new); } - - private byte[] parseChicory(byte[] source, int sourceLength, byte[] metadata) { - try (Prism prism = new Prism()) { - return prism.serialize(metadata, source, sourceLength); + private void evalScopesRecursive(StaticScope scope, ArrayList scopes) { + if (scope.getEnclosingScope() != null && scope.isBlockScope()) { + evalScopesRecursive(scope.getEnclosingScope(), scopes); } - } - // lineNumber (0-indexed) - private byte[] generateMetadata(String fileName, int lineNumber, Encoding encoding, DynamicScope scope, ParserType type) { - ByteList metadata = new ByteList(); - - // Filepath - byte[] name = fileName.getBytes(); - appendUnsignedInt(metadata, name.length); - metadata.append(name); - - // FIXME: I believe line number can be negative? - // Line Number (1-indexed) - appendUnsignedInt(metadata, lineNumber + 1); - - // Encoding - name = encoding.getName(); - appendUnsignedInt(metadata, name.length); - metadata.append(name); - - // frozen string literal - Boolean frozen = runtime.getInstanceConfig().isFrozenStringLiteral(); - metadata.append(frozen != null && frozen ? 1 : 0); - - // command-line flags - RubyInstanceConfig config = runtime.getInstanceConfig(); - byte flags = 0; - if (config.isSplit()) flags |= 1; // -a - if (config.isInlineScript()) flags |= 2; // -e - if (config.isProcessLineEnds()) flags |= 4; // -l - //if (config.isAssumeLoop()) flags |= 8; // -n - //if (config.isAssumePrinting()) flags |= 16; // -p - if (config.isXFlag()) flags |= 32; // -x - metadata.append(flags); - - // version - metadata.append(ParsingOptions.SyntaxVersion.V3_4.getValue()); - - // Do not lock encoding - metadata.append(0); - - // main script - metadata.append(1); - - // partial script - metadata.append(0); - - // freeze - metadata.append(0); - - // Eval scopes (or none for normal parses) - if (type == EVAL) { - encodeEvalScopes(metadata, scope.getStaticScope()); - } else { - appendUnsignedInt(metadata, 0); - } + scopes.add(new ParsingOptions.Scope( + Arrays + .stream(scope.getVariables()) + .map(String::getBytes) + .toArray(byte[][]::new), + IntStream + .range(0, scope.getVariables().length) + .mapToObj((i) -> ParsingOptions.Forwarding.NONE) + .toArray(ParsingOptions.Forwarding[]::new))); + } - return metadata.bytes(); // FIXME: extra arraycopy + private EnumSet commandLineFromConfig(RubyInstanceConfig config) { + var list = new ArrayList(); + + if (config.isSplit()) list.add(ParsingOptions.CommandLine.A); // -a + if (config.isInlineScript()) list.add(ParsingOptions.CommandLine.E); // -e + if (config.isProcessLineEnds()) list.add(ParsingOptions.CommandLine.L); // -l + if (config.isAssumeLoop()) list.add(ParsingOptions.CommandLine.N); // -n + if (config.isAssumePrinting()) list.add(ParsingOptions.CommandLine.P); // -p + if (config.isXFlag()) list.add(ParsingOptions.CommandLine.X); // -x + + return EnumSet.copyOf(list); } private void writeUnsignedInt(ByteList buf, int index, int value) { @@ -276,12 +248,17 @@ private void appendUnsignedInt(ByteList buf, int value) { buf.append(value >>> 24); } - private byte[] encodeEvalScopes(ByteList buf, StaticScope scope) { + private void encodeEvalScopes(ByteList buf, StaticScope scope) { int startIndex = buf.realSize(); + + // append uint 0 to reserve the space appendUnsignedInt(buf, 0); + + // write the scopes to the buffer int count = encodeEvalScopesInner(buf, scope, 1); + + // overwrite int 0 with scope count writeUnsignedInt(buf, startIndex, count); - return buf.bytes(); } private int encodeEvalScopesInner(ByteList buf, StaticScope scope, int count) { @@ -291,8 +268,13 @@ private int encodeEvalScopesInner(ByteList buf, StaticScope scope, int count) { // once more for method scope String names[] = scope.getVariables(); + + // number of variables appendUnsignedInt(buf, names.length); + + // forwarding flags buf.append(0); + for (String name : names) { // Get the bytes "raw" (which we use ISO8859_1 for) as this is how we record these in StaticScope. byte[] bytes = name.getBytes(ISO8859_1Encoding.INSTANCE.getCharset()); @@ -327,23 +309,23 @@ public IRubyObject getLineStub(ThreadContext context, ParseResult arg, int lineC @Override public ParseResult addGetsLoop(Ruby runtime, ParseResult result, boolean printing, boolean processLineEndings, boolean split) { var context = runtime.getCurrentContext(); - List newBody = new ArrayList<>(); + List newBody = new ArrayList<>(); if (processLineEndings) { - newBody.add(new Nodes.GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_BACKSLASH), + newBody.add(new GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_BACKSLASH), new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_SLASH)))); } - Nodes.GlobalVariableReadNode dollarUnderscore = new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, DOLLAR_UNDERSCORE)); + GlobalVariableReadNode dollarUnderscore = new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, DOLLAR_UNDERSCORE)); - List whileBody = new ArrayList<>(); + List whileBody = new ArrayList<>(); if (processLineEndings) { whileBody.add(new CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "chomp!"), null, null)); } if (split) { whileBody.add(new GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, "$F"), - new Nodes.CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "split"), null, null))); + new CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "split"), null, null))); } StatementsNode stmts = ((ProgramNode) result.getAST()).statements; @@ -362,7 +344,7 @@ public ParseResult addGetsLoop(Ruby runtime, ParseResult result, boolean printin nodes = new Node[newBody.size()]; newBody.toArray(nodes); - Nodes.ProgramNode newRoot = new Nodes.ProgramNode(-1, 0, 0, new RubySymbol[] {}, new StatementsNode(-1, 0, 0, nodes)); + ProgramNode newRoot = new ProgramNode(-1, 0, 0, new RubySymbol[] {}, new StatementsNode(-1, 0, 0, nodes)); ((ParseResultPrism) result).setRoot(newRoot); diff --git a/src/main/java/org/jruby/prism/parser/ParserPrismNative.java b/src/main/java/org/jruby/prism/parser/ParserPrismNative.java new file mode 100644 index 0000000..57bfa6c --- /dev/null +++ b/src/main/java/org/jruby/prism/parser/ParserPrismNative.java @@ -0,0 +1,33 @@ +package org.jruby.prism.parser; + +import org.jruby.Ruby; +import org.jruby.management.ParserStats; + +public class ParserPrismNative extends ParserPrismBase { + private final ParserBindingPrism prismLibrary; + + public ParserPrismNative(Ruby runtime, ParserBindingPrism prismLibrary) { + super(runtime); + this.prismLibrary = prismLibrary; + } + + protected byte[] parse(byte[] source, int sourceLength, byte[] metadata) { + long time = 0; + if (parserTiming) time = System.nanoTime(); + + ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary)); + prismLibrary.pm_buffer_init(buffer); + prismLibrary.pm_serialize_parse(buffer, source, sourceLength, metadata); + if (parserTiming) { + ParserStats stats = runtime.getParserManager().getParserStats(); + + stats.addPrismTimeCParseSerialize(System.nanoTime() - time); + } + + int length = buffer.length.intValue(); + byte[] src = new byte[length]; + buffer.value.get().get(0, src, 0, length); + + return src; + } +} diff --git a/src/main/java/org/jruby/prism/parser/ParserPrismWasm.java b/src/main/java/org/jruby/prism/parser/ParserPrismWasm.java new file mode 100644 index 0000000..4ce643d --- /dev/null +++ b/src/main/java/org/jruby/prism/parser/ParserPrismWasm.java @@ -0,0 +1,17 @@ +package org.jruby.prism.parser; + +import org.jruby.Ruby; +import org.prism.Prism; +import org.prism.PrismWASM; + +public class ParserPrismWasm extends ParserPrismBase { + private static final Prism prism = new PrismWASM(); + + public ParserPrismWasm(Ruby runtime) { + super(runtime); + } + + protected byte[] parse(byte[] source, int sourceLength, byte[] metadata) { + return prism.serialize(metadata, source, sourceLength); + } +}