Skip to content

Commit

Permalink
add MergeFiles & new globing dup_* directives
Browse files Browse the repository at this point in the history 
to handle -includeresource expressions like this, as suggested by @pkriens :

-includeresource @jar1.jar, @jar2.jar;dup_overwrite:=*, @jar3.jar;dup_skip:="META-INF/services/*,META-INF/MANIFEST.MF"

The value of these directives is a list of globs on the paths in the resource. Priority is probably merge (if plugin exists), overwrite, skip. Error/Warning should always be executed even if it matches the other ones.

Signed-off-by: Christoph Rueger <[email protected]>

rework to MergeFiles plugin
  • Loading branch information
@chrisrueger
chrisrueger committed Oct 15, 2024
1 parent 7c6bb03 commit 5df7631
Show file tree
Hide file tree
Showing 6 changed files with 214 additions and 62 deletions.
50 changes: 45 additions & 5 deletions biz.aQute.bndlib.tests/test/test/IncludeResourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.File;
Expand Down Expand Up @@ -185,31 +186,52 @@ private String testPreprocessing(String ir, String resource, String... checks) t
}

@Test
public void testIncludeResourceAppendDuplicates() throws Exception {
public void testIncludeResourceDuplicatesMerge() throws Exception {

try (Builder a = new Builder();) {
a.addClasspath(new File("jar/jarA.jar"));
a.addClasspath(a.getFile("jar/jarB.jar"));
a.setIncludeResource(
"@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;:duplicates:=APPEND");
"@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;dup_merge:=*");
Jar jar = a.build();
assertTrue(a.check());

assertTrue(jar.getDirectories()
.containsKey("META-INF/services"));

Resource resource = jar.getResource("META-INF/services/foo");
assertEquals("ab", IO.collect(resource.openInputStream()));
assertEquals("a\nb", IO.collect(resource.openInputStream()));

}
}

@Test
public void testIncludeResourceAppendDuplicatesLiteral(@InjectTemporaryDirectory
public void testIncludeResourceDuplicatesError() throws Exception {

try (Builder a = new Builder();) {
a.addClasspath(new File("jar/jarA.jar"));
a.addClasspath(a.getFile("jar/jarB.jar"));
a.setIncludeResource("@jar/jarA.jar!/META-INF/services/*, @jar/jarB.jar!/META-INF/services/*;dup_error:=*");
Jar jar = a.build();
assertFalse(a.check());
assertEquals("Duplicate file overwritten: META-INF/services/foo", a.getErrors()
.get(0));

assertTrue(jar.getDirectories()
.containsKey("META-INF/services"));

Resource resource = jar.getResource("META-INF/services/foo");
assertEquals("b", IO.collect(resource.openInputStream()));

}
}

@Test
public void testIncludeResourceLiteralDuplicatesMerge(@InjectTemporaryDirectory
File tmp) throws Exception {

try (Builder b = new Builder()) {
b.setIncludeResource("/a/a.txt;literal='a', /a/a.txt;literal='b';:duplicates:=APPEND");
b.setIncludeResource("/a/a.txt;literal='a', /a/a.txt;literal='b';dup_merge:=*");
b.build();
assertTrue(b.check());

Expand All @@ -219,4 +241,22 @@ public void testIncludeResourceAppendDuplicatesLiteral(@InjectTemporaryDirectory
assertEquals("ab", IO.collect(IO.getFile(tmp, "a/a.txt")));
}
}

@Test
public void testIncludeResourceLiteralDuplicatesError(@InjectTemporaryDirectory
File tmp) throws Exception {

try (Builder b = new Builder()) {
b.setIncludeResource("/a/a.txt;literal='a', /a/a.txt;literal='b';dup_error:=*");
b.build();
assertFalse(b.check());
assertEquals("Duplicate file overwritten: /a/a.txt", b.getErrors()
.get(0));

b.getJar()
.writeFolder(tmp);

assertEquals("b", IO.collect(IO.getFile(tmp, "a/a.txt")));
}
}
}
130 changes: 117 additions & 13 deletions biz.aQute.bndlib/src/aQute/bnd/osgi/Builder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.SequenceInputStream;
import java.net.URI;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand All @@ -26,6 +28,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
Expand All @@ -42,6 +45,7 @@
import aQute.bnd.cdi.CDIAnnotations;
import aQute.bnd.component.DSAnnotations;
import aQute.bnd.differ.DiffPluginImpl;
import aQute.bnd.exceptions.Exceptions;
import aQute.bnd.header.Attrs;
import aQute.bnd.header.OSGiHeader;
import aQute.bnd.header.Parameters;
Expand All @@ -55,7 +59,7 @@
import aQute.bnd.metatype.MetatypeAnnotations;
import aQute.bnd.osgi.Descriptors.PackageRef;
import aQute.bnd.osgi.Descriptors.TypeRef;
import aQute.bnd.osgi.Jar.DupStrategy;
import aQute.bnd.osgi.metainf.MetaInfServiceMerger;
import aQute.bnd.osgi.metainf.MetaInfServiceParser;
import aQute.bnd.plugin.jpms.JPMSAnnotations;
import aQute.bnd.plugin.jpms.JPMSModuleInfoPlugin;
Expand All @@ -66,6 +70,7 @@
import aQute.bnd.service.diff.Diff;
import aQute.bnd.service.diff.Tree;
import aQute.bnd.service.diff.Type;
import aQute.bnd.service.merge.MergeFiles;
import aQute.bnd.service.specifications.BuilderSpecification;
import aQute.bnd.stream.MapStream;
import aQute.bnd.unmodifiable.Maps;
Expand All @@ -78,6 +83,7 @@
import aQute.lib.strings.Strings;
import aQute.lib.zip.ZipUtil;
import aQute.libg.generics.Create;
import aQute.libg.glob.Glob;
import aQute.libg.re.RE;

/**
Expand Down Expand Up @@ -1377,8 +1383,6 @@ public boolean addAll(Jar to, Jar sub, Instruction filter, String destination) {
private boolean addAll(Jar to, Jar sub, Instruction filter, String destination, Function<String, String> modifier,
Map<String, String> extra) {

DupStrategy dupStrategy = dupStrategy(extra);

boolean dupl = false;
for (String name : sub.getResources()
.keySet()) {
Expand All @@ -1390,8 +1394,19 @@ private boolean addAll(Jar to, Jar sub, Instruction filter, String destination,

if (filter == null || filter.matches(name) ^ filter.isNegated()) {

dupl |= to.putResource(Processor.appendPath(destination, modifier.apply(name)), sub.getResource(name),
dupStrategy);
String path = Processor.appendPath(destination, modifier.apply(name));
Resource resource = sub.getResource(name);
Resource existing = to.getResource(path);
boolean duplicate = existing != null;

if (!duplicate) {
dupl |= to.putResource(path, resource, true);
} else {
Optional<Resource> maybeMerged = DupStrategy.onDuplicate(path, existing, resource, extra, this);
if (maybeMerged.isPresent()) {
dupl |= to.putResource(path, maybeMerged.get());
}
}

}
}
Expand Down Expand Up @@ -1433,9 +1448,18 @@ private void copy(Jar jar, String path, File from, Instructions preprocess, Map<
}

private void copy(Jar jar, String path, Resource resource, Map<String, String> extra) {
DupStrategy dupStrategy = dupStrategy(extra);
Resource existing = jar.getResource(path);

boolean duplicate = existing != null;
if (!duplicate) {
jar.putResource(path, resource, true);
} else {
Optional<Resource> maybeMerged = DupStrategy.onDuplicate(path, existing, resource, extra, this);
if (maybeMerged.isPresent()) {
jar.putResource(path, maybeMerged.get(), true);
}
}

jar.putResource(path, resource, dupStrategy);
if (isTrue(extra.get(LIB_DIRECTIVE))) {
setProperty(BUNDLE_CLASSPATH, append(getProperty(BUNDLE_CLASSPATH, "."), path));
}
Expand Down Expand Up @@ -1775,6 +1799,28 @@ public Pattern getDoNotCopy() {
static SPIDescriptorGenerator spiDescriptorGenerator = new SPIDescriptorGenerator();
static JPMSMultiReleasePlugin jpmsReleasePlugin = new JPMSMultiReleasePlugin();
static MetaInfServiceParser metaInfoServiceParser = new MetaInfServiceParser();
static MetaInfServiceMerger metaInfoServiceMerger = new MetaInfServiceMerger();
static MergeFiles defaultResourceMerger = new MergeFiles() {

@Override
public Optional<Resource> merge(String path, Resource a,
Resource b) {

try (
SequenceInputStream in = new SequenceInputStream(
a.openInputStream(), b.openInputStream())) {

long lastModified = Math.max(a.lastModified(),
b.lastModified());
return Optional.of(new EmbeddedResource(
ByteBuffer.wrap(in.readAllBytes()),
lastModified));
} catch (Exception e) {
throw Exceptions.duck(e);
}

}
};

@Override
protected void setTypeSpecificPlugins(PluginsContainer pluginsContainer) {
Expand All @@ -1789,6 +1835,8 @@ protected void setTypeSpecificPlugins(PluginsContainer pluginsContainer) {
pluginsContainer.add(spiDescriptorGenerator);
pluginsContainer.add(jpmsReleasePlugin);
pluginsContainer.add(metaInfoServiceParser);
pluginsContainer.add(metaInfoServiceMerger);
pluginsContainer.add(defaultResourceMerger);
super.setTypeSpecificPlugins(pluginsContainer);
}

Expand Down Expand Up @@ -2095,11 +2143,67 @@ public String system(boolean allowFail, String command, String input) throws IOE
return cachedSystemCalls.computeIfAbsent(key, asFunction(k -> super.system(allowFail, command, input)));
}

private DupStrategy dupStrategy(Map<String, String> extra) {
String val = extra.get(":duplicates:");
DupStrategy dupStrategy = val != null ? DupStrategy.valueOf(val.trim()
.toUpperCase())
: DupStrategy.OVERWRITE;
return dupStrategy;




/**
* Helper for handling inluderesource duplicates.
*/
private final class DupStrategy {

private static final String DUP_MSG = "Duplicate file overwritten: %s";

private static Optional<Resource> onDuplicate(String path, Resource existing, Resource resource,
Map<String, String> extra,
Processor proc) {
// The value of these directives is a list of globs on the paths in
// the resource.
String dup_overwrite = extra.get("dup_overwrite:");
String dup_merge = extra.get("dup_merge:");
String dup_error = extra.get("dup_error:");
String dup_warning = extra.get("dup_warning:");
String dup_skip = extra.get("dup_skip:");


if (matches(path, dup_error)) {
proc.error(DUP_MSG, path);
}
if (matches(path, dup_warning)) {
proc.warning(DUP_MSG, path);
}

if (matches(path, dup_merge)) {

return proc.getPlugins(MergeFiles.class)
.stream()
.map(mf -> mf.merge(path, existing, resource))
.filter(Optional::isPresent)
.findFirst()
.orElse(Optional.of(resource));

} else if (matches(path, dup_overwrite)) {
return Optional.ofNullable(resource);
} else if (matches(path, dup_skip)) {
return Optional.ofNullable(null);
}


return Optional.ofNullable(resource);
}

private static boolean matches(String path, String globs) {
if(globs == null) {
return false;
}

// default is '*' if blank
if (globs.isBlank() || globs.trim()
.equals("*")) {
return Glob.ALL.matches(path);
}

return Stream.of(globs.split(",")).anyMatch(glob -> new Glob(glob).matches(path));
}
}
}
46 changes: 2 additions & 44 deletions biz.aQute.bndlib/src/aQute/bnd/osgi/Jar.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.SequenceInputStream;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.file.FileVisitOption;
Expand Down Expand Up @@ -72,26 +71,6 @@
import aQute.libg.glob.PathSet;

public class Jar implements Closeable {

/**
* Controls how duplicate resources are handled which are put into a jar
* file.
*/
public enum DupStrategy {
/**
* default: overwrite existing
*/
OVERWRITE,
/**
* append (concatenate) the new to the existing
*/
APPEND,
/**
* skip if already exist
*/
NOTHING
}

private static final int BUFFER_SIZE = IOConstants.PAGE_SIZE * 16;
/**
* Note that setting the January 1st 1980 (or even worse, "0", as time)
Expand Down Expand Up @@ -379,10 +358,6 @@ public boolean putResource(String path, Resource resource) {
}

public boolean putResource(String path, Resource resource, boolean overwrite) {
return putResource(path, resource, overwrite ? DupStrategy.OVERWRITE : DupStrategy.NOTHING);
}

public boolean putResource(String path, Resource resource, DupStrategy strategy) {
check();
path = ZipUtil.cleanPath(path);

Expand All @@ -404,29 +379,12 @@ public boolean putResource(String path, Resource resource, DupStrategy strategy)
directories.put(dir, null);
}
}

Resource existing = s.get(path);
boolean duplicate = existing != null;
if (!duplicate || DupStrategy.OVERWRITE == strategy) {
boolean duplicate = s.containsKey(path);
if (!duplicate || overwrite) {
resources.put(path, resource);
s.put(path, resource);
updateModified(resource.lastModified(), path);
}
else if (duplicate && DupStrategy.APPEND == strategy) {
try (SequenceInputStream in = new SequenceInputStream(existing.openInputStream(),
resource.openInputStream());) {

long lastModified = Math.max(existing.lastModified(), resource.lastModified());
Resource r = new EmbeddedResource(ByteBuffer.wrap(in.readAllBytes()),
lastModified);
// addExtra(r, extra.get("extra"));
resources.put(path, r);
s.put(path, r);
updateModified(resource.lastModified(), path);
} catch (Exception e) {
throw Exceptions.duck(e);
}
}
return duplicate;
}

Expand Down
Loading

0 comments on commit 5df7631

Please sign in to comment.