Skip to content

Formally add WASM support with Chicory #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
headius wants to merge 9 commits into
base: topic/chicory
Choose a base branch
from
Open

Formally add WASM support with Chicory #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
745df28
Use a release version of JRuby
headius Jan 22, 2026
548b3a1
Use new chicory-prism artifact
headius Jan 22, 2026
aa7627f
Move release plugins to release profile
headius Jan 22, 2026
421936a
Bump version to 2.0 snapshot
headius Jan 22, 2026
2845e01
Separate native and wasm parsers
headius Jan 22, 2026
2408aa8
Document some metadata setup
headius Jan 25, 2026
0d986cc
Bump up to v4.0
headius Jan 25, 2026
b094616
Clean up unused imports
headius Jan 25, 2026
cc7e780
Re-use shared WASM Prism instance
headius Jan 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 56 additions & 44 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<groupId>org.jruby</groupId>
<artifactId>jruby-prism</artifactId>
<packaging>jar</packaging>
<version>1.5.0</version>
<version>2.0.0-SNAPSHOT</version>
<name>jruby-prism</name>
<description>
Java portion of JRuby Prism parser support.
Expand Down Expand Up @@ -66,12 +66,12 @@
<dependency>
<groupId>org.jruby</groupId>
<artifactId>jruby-base</artifactId>
<version>10.0.0.0-SNAPSHOT</version>
<version>10.0.2.0</version>
</dependency>
<dependency>
<groupId>com.prism</groupId>
<artifactId>java-prism</artifactId>
<version>999-SNAPSHOT</version>
<groupId>org.jruby</groupId>
<artifactId>chicory-prism</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
</dependencies>

Expand Down Expand Up @@ -116,45 +116,6 @@
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.5</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
<configuration>
<doclint>none</doclint>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
Expand Down Expand Up @@ -210,5 +171,56 @@
</plugins>
</build>
</profile>
<profile>
<id>release</id>
<build>
<plugins>
<plugin>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.6</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
<configuration>
<gpgArguments>
<gpgArgument>--pinentry-mode</gpgArgument>
<gpgArgument>loopback</gpgArgument>
</gpgArguments>
</configuration>
</plugin>
<plugin>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
<configuration>
<doclint>none</doclint>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
34 changes: 23 additions & 11 deletions src/main/java/org/jruby/prism/ParserProviderPrism.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,43 @@
package org.jruby.prism;

import jnr.ffi.LibraryLoader;
import org.jruby.Ruby;
import org.jruby.ir.builder.IRBuilderFactory;
import org.jruby.parser.Parser;
import org.jruby.parser.ParserManager;
import org.jruby.parser.ParserProvider;
import org.jruby.prism.parser.ParserPrism;
import org.jruby.prism.parser.ParserBindingPrism;
import org.jruby.prism.builder.IRBuilderFactoryPrism;
import org.jruby.prism.parser.ParserBindingPrism;
import org.jruby.prism.parser.ParserPrismNative;
import org.jruby.prism.parser.ParserPrismWasm;

import jnr.ffi.LibraryLoader;
import java.io.File;

public class ParserProviderPrism implements ParserProvider {
private static ParserBindingPrism prismLibrary;

public void initialize(String path) {
if (prismLibrary != null) {
System.out.println("Prism already initialized");
return;
if (new File(path).exists()) {
if (prismLibrary != null) {
System.out.println("Prism already initialized");
return;
}
prismLibrary = LibraryLoader.create(ParserBindingPrism.class).load(path);
// We do something extra here as a side-effect which is how we get an UnsatisfiedLinkError
// If the library didn't in fact find the .so or has other loading problems.
ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary));
} else {
prismLibrary = null;
}
prismLibrary = LibraryLoader.create(ParserBindingPrism.class).load(path);
// We do something extra here as a side-effect which is how we get an UnsatisfiedLinkError
// If the library didn't in fact find the .so or has other loading problems.
ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary));
}

public Parser getParser(Ruby runtime) {
return new ParserPrism(runtime, prismLibrary);
if (ParserManager.PARSER_WASM || prismLibrary == null) {
// uninitialized dynamic lib or wasm requested
return new ParserPrismWasm(runtime);
}

return new ParserPrismNative(runtime, prismLibrary);
}

public IRBuilderFactory getBuilderFactory() {
Expand Down
86 changes: 35 additions & 51 deletions ...a/org/jruby/prism/parser/ParserPrism.java → ...g/jruby/prism/parser/ParserPrismBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import org.jruby.ext.coverage.CoverageData;
import org.jruby.management.ParserStats;
import org.jruby.parser.Parser;
import org.jruby.parser.ParserManager;
import org.jruby.parser.ParserType;
import org.jruby.parser.StaticScope;
import org.jruby.runtime.DynamicScope;
Expand All @@ -21,10 +20,16 @@
import org.jruby.util.ByteList;
import org.jruby.util.CommonByteLists;
import org.jruby.util.io.ChannelHelper;
import org.prism.Nodes;
import org.prism.Nodes.*;
import org.prism.Nodes.ArgumentsNode;
import org.prism.Nodes.CallNode;
import org.prism.Nodes.CallNodeFlags;
import org.prism.Nodes.GlobalVariableReadNode;
import org.prism.Nodes.GlobalVariableWriteNode;
import org.prism.Nodes.Node;
import org.prism.Nodes.ProgramNode;
import org.prism.Nodes.StatementsNode;
import org.prism.Nodes.WhileNode;
import org.prism.ParsingOptions;
import org.prism.Prism;

import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
Expand All @@ -39,16 +44,15 @@
import static org.jruby.parser.ParserType.EVAL;
import static org.jruby.parser.ParserType.MAIN;

public class ParserPrism extends Parser {
private boolean parserTiming = org.jruby.util.cli.Options.PARSER_SUMMARY.load();
public abstract class ParserPrismBase extends Parser {
protected boolean parserTiming = org.jruby.util.cli.Options.PARSER_SUMMARY.load();

private final ParserBindingPrism prismLibrary;

public ParserPrism(Ruby runtime, ParserBindingPrism prismLibrary) {
public ParserPrismBase(Ruby runtime) {
super(runtime);
this.prismLibrary = prismLibrary;
}

protected abstract byte[] parse(byte[] source, int sourceLength, byte[] metadata);

@Override
public ParseResult parse(String fileName, int lineNumber, ByteList content, DynamicScope existingScope, ParserType type) {
int sourceLength = content.realSize();
Expand Down Expand Up @@ -113,10 +117,10 @@ private ParseResult parseInternal(String fileName, DynamicScope blockScope, byte
coverageMode = runtime.getCoverageData().getMode();
}

ParseResultPrism result = new ParseResultPrism(fileName, source, (Nodes.ProgramNode) res.value, res.source, encoding, coverageMode);
ParseResultPrism result = new ParseResultPrism(fileName, source, (ProgramNode) res.value, res.source, encoding, coverageMode);
if (blockScope != null) {
if (type == MAIN) { // update TOPLEVEL_BINDNG
RubySymbol[] locals = ((Nodes.ProgramNode) result.getAST()).locals;
RubySymbol[] locals = ((ProgramNode) result.getAST()).locals;
for (int i = 0; i < locals.length; i++) {
blockScope.getStaticScope().addVariableThisScope(locals[i].idString());
}
Expand Down Expand Up @@ -174,36 +178,6 @@ private byte[] loadFully(String fileName, InputStream in) {
}
}


private byte[] parse(byte[] source, int sourceLength, byte[] metadata) {
if (ParserManager.PARSER_WASM) return parseChicory(source, sourceLength, metadata);

long time = 0;
if (parserTiming) time = System.nanoTime();

ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary));
prismLibrary.pm_buffer_init(buffer);
prismLibrary.pm_serialize_parse(buffer, source, sourceLength, metadata);
if (parserTiming) {
ParserStats stats = runtime.getParserManager().getParserStats();

stats.addPrismTimeCParseSerialize(System.nanoTime() - time);
}

int length = buffer.length.intValue();
byte[] src = new byte[length];
buffer.value.get().get(0, src, 0, length);

return src;
}


private byte[] parseChicory(byte[] source, int sourceLength, byte[] metadata) {
try (Prism prism = new Prism()) {
return prism.serialize(metadata, source, sourceLength);
}
}

// lineNumber (0-indexed)
private byte[] generateMetadata(String fileName, int lineNumber, Encoding encoding, DynamicScope scope, ParserType type) {
ByteList metadata = new ByteList();
Expand Down Expand Up @@ -238,7 +212,7 @@ private byte[] generateMetadata(String fileName, int lineNumber, Encoding encodi
metadata.append(flags);

// version
metadata.append(ParsingOptions.SyntaxVersion.V3_4.getValue());
metadata.append(ParsingOptions.SyntaxVersion.V4_0.getValue());

// Do not lock encoding
metadata.append(0);
Expand Down Expand Up @@ -276,12 +250,17 @@ private void appendUnsignedInt(ByteList buf, int value) {
buf.append(value >>> 24);
}

private byte[] encodeEvalScopes(ByteList buf, StaticScope scope) {
private void encodeEvalScopes(ByteList buf, StaticScope scope) {
int startIndex = buf.realSize();

// append uint 0 to reserve the space
appendUnsignedInt(buf, 0);

// write the scopes to the buffer
int count = encodeEvalScopesInner(buf, scope, 1);

// overwrite int 0 with scope count
writeUnsignedInt(buf, startIndex, count);
return buf.bytes();
}

private int encodeEvalScopesInner(ByteList buf, StaticScope scope, int count) {
Expand All @@ -291,8 +270,13 @@ private int encodeEvalScopesInner(ByteList buf, StaticScope scope, int count) {

// once more for method scope
String names[] = scope.getVariables();

// number of variables
appendUnsignedInt(buf, names.length);

// forwarding flags
buf.append(0);

for (String name : names) {
// Get the bytes "raw" (which we use ISO8859_1 for) as this is how we record these in StaticScope.
byte[] bytes = name.getBytes(ISO8859_1Encoding.INSTANCE.getCharset());
Expand Down Expand Up @@ -327,23 +311,23 @@ public IRubyObject getLineStub(ThreadContext context, ParseResult arg, int lineC
@Override
public ParseResult addGetsLoop(Ruby runtime, ParseResult result, boolean printing, boolean processLineEndings, boolean split) {
var context = runtime.getCurrentContext();
List<Nodes.Node> newBody = new ArrayList<>();
List<Node> newBody = new ArrayList<>();

if (processLineEndings) {
newBody.add(new Nodes.GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_BACKSLASH),
newBody.add(new GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_BACKSLASH),
new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, CommonByteLists.DOLLAR_SLASH))));
}

Nodes.GlobalVariableReadNode dollarUnderscore = new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, DOLLAR_UNDERSCORE));
GlobalVariableReadNode dollarUnderscore = new GlobalVariableReadNode(-1, 0, 0, asSymbol(context, DOLLAR_UNDERSCORE));

List<Nodes.Node> whileBody = new ArrayList<>();
List<Node> whileBody = new ArrayList<>();

if (processLineEndings) {
whileBody.add(new CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "chomp!"), null, null));
}
if (split) {
whileBody.add(new GlobalVariableWriteNode(-1, 0, 0, asSymbol(context, "$F"),
new Nodes.CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "split"), null, null)));
new CallNode(-1, 0, 0, (short) 0, dollarUnderscore, asSymbol(context, "split"), null, null)));
}

StatementsNode stmts = ((ProgramNode) result.getAST()).statements;
Expand All @@ -362,7 +346,7 @@ public ParseResult addGetsLoop(Ruby runtime, ParseResult result, boolean printin

nodes = new Node[newBody.size()];
newBody.toArray(nodes);
Nodes.ProgramNode newRoot = new Nodes.ProgramNode(-1, 0, 0, new RubySymbol[] {}, new StatementsNode(-1, 0, 0, nodes));
ProgramNode newRoot = new ProgramNode(-1, 0, 0, new RubySymbol[] {}, new StatementsNode(-1, 0, 0, nodes));

((ParseResultPrism) result).setRoot(newRoot);

Expand Down
33 changes: 33 additions & 0 deletions src/main/java/org/jruby/prism/parser/ParserPrismNative.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package org.jruby.prism.parser;

import org.jruby.Ruby;
import org.jruby.management.ParserStats;

public class ParserPrismNative extends ParserPrismBase {
private final ParserBindingPrism prismLibrary;

public ParserPrismNative(Ruby runtime, ParserBindingPrism prismLibrary) {
super(runtime);
this.prismLibrary = prismLibrary;
}

protected byte[] parse(byte[] source, int sourceLength, byte[] metadata) {
long time = 0;
if (parserTiming) time = System.nanoTime();

ParserBindingPrism.Buffer buffer = new ParserBindingPrism.Buffer(jnr.ffi.Runtime.getRuntime(prismLibrary));
prismLibrary.pm_buffer_init(buffer);
prismLibrary.pm_serialize_parse(buffer, source, sourceLength, metadata);
if (parserTiming) {
ParserStats stats = runtime.getParserManager().getParserStats();

stats.addPrismTimeCParseSerialize(System.nanoTime() - time);
}

int length = buffer.length.intValue();
byte[] src = new byte[length];
buffer.value.get().get(0, src, 0, length);

return src;
}
}
Loading