diff --git a/pom.xml b/pom.xml
index 9dc8db8..8300a25 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
http://maven.apache.org/xsd/maven-4.0.0.xsd">
4.0.0
ldbc.snb
- bteronh
+ bteronhplus
0.0.3
jar
@@ -78,7 +78,12 @@
com.beust
jcommander
1.48
-
+
+
+ org.jgrapht
+ jgrapht-core
+ 1.1.0
+
diff --git a/src/main/java/ldbc/snb/bteronh/BTERMain.java b/src/main/java/ldbc/snb/bteronh/BTERMain.java
deleted file mode 100644
index bf7fcef..0000000
--- a/src/main/java/ldbc/snb/bteronh/BTERMain.java
+++ /dev/null
@@ -1,69 +0,0 @@
-package ldbc.snb.bteronh;
-
-import com.beust.jcommander.JCommander;
-import ldbc.snb.bteronh.hadoop.HadoopBTERGenerator;
-import ldbc.snb.bteronh.structures.Arguments;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.Properties;
-
-public class BTERMain {
- public static void main(String [] args) {
-
- System.out.println("Generating edges");
- Configuration conf = new Configuration();
- conf.setInt("ldbc.snb.bteronh.generator.numThreads",1);
- conf.setLong("ldbc.snb.bteronh.generator.numNodes",10000);
- conf.setInt("ldbc.snb.bteronh.generator.seed",12323540);
- conf.set("ldbc.snb.bteronh.serializer.workspace","./");
- conf.set("ldbc.snb.bteronh.generator.degreeSequence","src/main/resources/degreeSequences/dblp");
- conf.set("ldbc.snb.bteronh.generator.ccPerDegree","src/main/resources/CCs/dblp");
-
-
- Arguments arguments = new Arguments();
- new JCommander(arguments,args);
- for(Arguments.Property p : arguments.getProperties()) {
- conf.set(p.getProperty(),p.getValue());
- }
-
- for(String propertyFile : arguments.getPropertyFiles()) {
- Properties properties = new Properties();
- try {
- properties.load(new FileReader(propertyFile));
- } catch(IOException e) {
- e.printStackTrace();
- System.exit(1);
- }
- for( String s : properties.stringPropertyNames()) {
- conf.set(s,properties.getProperty(s));
- }
- }
-
-
- System.out.println("ldbc.snb.bteronh.generator.numThreads "+conf.get("ldbc.snb.bteronh.generator.numThreads"));
- System.out.println("ldbc.snb.bteronh.generator.numNodes "+conf.get("ldbc.snb.bteronh.generator.numNodes"));
- System.out.println("ldbc.snb.bteronh.generator.seed "+conf.get("ldbc.snb.bteronh.generator.seed"));
- System.out.println("ldbc.snb.bteronh.serializer.workspace "+conf.get("ldbc.snb.bteronh.serializer.workspace"));
- System.out.println("ldbc.snb.bteronh.generator.degreeSequence "+conf.get("ldbc.snb.bteronh.generator" +
- ".degreeSequence"));
- System.out.println("ldbc.snb.bteronh.generator.ccPerDegree "+conf.get("ldbc.snb.bteronh.generator" +
- ".ccPerDegree"));
-
- try {
- System.out.println("Starting execution");
- long start = System.currentTimeMillis();
- HadoopBTERGenerator bterGenerator = new HadoopBTERGenerator(conf);
- bterGenerator.run();
- long end = System.currentTimeMillis();
- System.out.println("Execution time: "+(end-start)/1000.0+" second");
- } catch(Exception e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- }
-}
diff --git a/src/main/java/ldbc/snb/bteronh/algorithms/Algorithms.java b/src/main/java/ldbc/snb/bteronh/algorithms/Algorithms.java
deleted file mode 100644
index 2085c9a..0000000
--- a/src/main/java/ldbc/snb/bteronh/algorithms/Algorithms.java
+++ /dev/null
@@ -1,158 +0,0 @@
-package ldbc.snb.bteronh.algorithms;
-
-import javafx.util.Pair;
-import ldbc.snb.bteronh.structures.BTERStats;
-import ldbc.snb.bteronh.structures.Edge;
-import umontreal.iro.lecuyer.probdist.EmpiricalDist;
-import umontreal.iro.lecuyer.randvar.RandomVariateGen;
-import umontreal.iro.lecuyer.rng.LFSR113;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.*;
-
-/**
- * Created by aprat on 14/07/16.
- */
-public class Algorithms {
- public static int BinarySearch(ArrayList> array, Long degree) {
-
- /*int min = 0;
- int max = array.size();
- while(min <= max) {
- int midPoint = (max - min) / 2 + min;
- if(midPoint >= array.size()) return array.size()-1;
- if(midPoint < 0) return 0;
- if(array.get(midPoint).getKey() > degree ) {
- max = midPoint - 1;
- } else if(array.get(midPoint).getKey() < degree) {
- min = midPoint + 1;
- } else {
- return midPoint;
- }
- }
- return max;
- */
- int pos = Collections.binarySearch(array, new Pair(degree, 0.0), new Comparator> ( ){
-
- @Override
- public int compare(Pair o1, Pair o2) {
- if(o1.getKey() < o2.getKey()) return -1;
- return 1;
- }
- });
-
- if(pos < 0) {
- return -(pos+1);
- }
- return pos;
- }
-
- public static RandomVariateGen GetDegreeSequenceSampler(List observedSequence, long numNodes, int seed ) {
-
- System.out.println("Creating sampler for degree sequence generation");
- observedSequence.sort( new Comparator() {
- @Override
- public int compare(Integer o1, Integer o2) {
- return o1 - o2;
- }
- });
- double [] sequence = new double[observedSequence.size()];
- for(int i = 0; i < observedSequence.size(); ++i){
- sequence[i] = observedSequence.get(i);
- }
- EmpiricalDist degreeDistribution = new EmpiricalDist(sequence);
-
-
- LFSR113 random = new LFSR113();
- int [] seeds = new int[4];
- seeds[0] = 128+seed;
- seeds[1] = 128+seed;
- seeds[2] = 128+seed;
- seeds[3] = 128+seed;
- //LFSR113.setPackageSeed(seeds);
- random.setSeed(seeds);
- RandomVariateGen randomVariateGen = new RandomVariateGen(random,degreeDistribution);
- return randomVariateGen;
- }
-
- public static double [] GenerateCCperDegree( ArrayList> ccDistribution, int maxDegree) {
-
- System.out.println("Loading CC distribution");
- double [] cc = new double[maxDegree+1];
- cc[0] = 0.0;
- cc[1] = 0.0;
- for(int i = 2; i < maxDegree+1; ++i) {
- int degree = i;
- int pos = Algorithms.BinarySearch(ccDistribution,(long)degree);
- if(ccDistribution.get(pos).getKey() == degree || pos == (ccDistribution.size() - 1)) {
- cc[degree] = ccDistribution.get(pos).getValue();
- } else if( pos < ccDistribution.size() - 1 ){
- long min = ccDistribution.get(pos).getKey();
- long max = ccDistribution.get(pos+1).getKey();
- double ratio = (degree - min) / (double)(max - min);
- double minCC = ccDistribution.get(pos).getValue();
- double maxCC = ccDistribution.get(pos+1).getValue();
- double cc_current = ratio * (maxCC - minCC ) + minCC;
- cc[degree] = cc_current;
- }
- }
- return cc;
- }
-
-
-
- public static int SampleCumulative(double [] cumulative, Random random) {
- double randomDis = random.nextDouble();
- int res = Arrays.binarySearch(cumulative, randomDis);
- if(res < 0) {
- return -(res+1);
- }
- return res;
- }
-
- public static Edge BTERSample(BTERStats stats, Random random) throws IOException {
-
- long totalWeight = stats.getWeightPhase1()+stats.getWeightPhase2();
- double prob = random.nextDouble();
- if(prob < stats.getWeightPhase1()/(double)(totalWeight)) {
- return BTERSamplePhase1(stats,random);
- }
- return BTERSamplePhase2(stats,random);
- }
-
-
- public static Edge BTERSamplePhase1(BTERStats stats, Random random) throws IOException{
- int group = SampleCumulative(stats.getCumulativeGroups(),random);
- double r1 = random.nextDouble();
- long offset = (stats.getGroupIndex(group) + (long)Math.floor(r1*stats.getGroupNumBuckets(group))*stats.getGroupBucketSize(group));
- double r2 = random.nextDouble();
- long firstNode = (long)Math.floor(r2*stats.getGroupBucketSize(group)) + offset;
- double r3 = random.nextDouble();
- long secondNode = (long)Math.floor(r3*(stats.getGroupBucketSize(group)-1)) + offset;
- if( secondNode >= firstNode ) {
- secondNode+=1;
- }
- return new Edge(firstNode,secondNode);
- }
-
- public static Edge BTERSamplePhase2(BTERStats stats, Random random) throws IOException {
- long firstNode = BTERSampleNodePhase2(stats, random);
- long secondNode = BTERSampleNodePhase2(stats, random);
- return new Edge(firstNode, secondNode);
- }
-
- public static long BTERSampleNodePhase2(BTERStats stats, Random random) {
- int degree = SampleCumulative(stats.getCumulativeDegrees(),random);
- double r1 = random.nextDouble();
- double r2 = random.nextDouble();
- if(r1 < stats.getDegreeWeightRatio(degree)) {
- return (long)Math.floor(r2*stats.getDegreeNFill(degree)) + stats.getDegreeIndex(degree);
- } else {
- return (long)Math.floor(r2*(stats.getDegreeN(degree) - stats.getDegreeNFill(degree))) + stats.getDegreeIndex(degree) + stats.getDegreeNFill(degree);
- }
- }
-}
diff --git a/src/main/java/ldbc/snb/bteronh/hadoop/HadoopBTERGenerator.java b/src/main/java/ldbc/snb/bteronh/hadoop/HadoopBTERGenerator.java
deleted file mode 100644
index 70e429e..0000000
--- a/src/main/java/ldbc/snb/bteronh/hadoop/HadoopBTERGenerator.java
+++ /dev/null
@@ -1,204 +0,0 @@
-package ldbc.snb.bteronh.hadoop;
-
-import javafx.util.Pair;
-import ldbc.snb.bteronh.algorithms.Algorithms;
-import ldbc.snb.bteronh.structures.BTERStats;
-import ldbc.snb.bteronh.structures.Edge;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-/**
- * Created by aprat on 16/08/16.
- */
-public class HadoopBTERGenerator {
-
- protected static String isHDFSPath(String fileName) {
- if (fileName.startsWith("hdfs://")) {
- return fileName.substring(7);
- }
- return null;
- }
-
- protected static String isLocalPath(String fileName) {
- if (fileName.startsWith("file://")) {
- return fileName.substring(7);
- }
- return null;
- }
-
- public static class HadoopBTERGeneratorMapper extends Mapper {
-
- private BufferedReader getFile(String fileName, Configuration conf) throws IOException {
- String realFileName;
- if ((realFileName = isHDFSPath(fileName)) != null) {
- FileSystem fs = FileSystem.get(conf);
- return new BufferedReader( new InputStreamReader(fs.open( new Path(realFileName))));
- } else if((realFileName = isLocalPath(fileName)) != null) {
- return new BufferedReader(new FileReader(realFileName));
- } else {
- throw new IOException("Invalid file URI "+fileName+". It must start with hdfs:// or file://");
- }
- }
-
- @Override
- public void map(LongWritable key, Text value, Context context)
- throws IOException, InterruptedException {
-
- Configuration conf = context.getConfiguration();
- int threadId = Integer.parseInt(value.toString());
- int seed = conf.getInt("ldbc.snb.bteronh.generator.seed",0);
- int numThreads = conf.getInt("ldbc.snb.bteronh.generator.numThreads",1);
- long numNodes = conf.getLong("ldbc.snb.bteronh.generator.numNodes",10000);
- String degreeSequenceFile = conf.get("ldbc.snb.bteronh.generator.degreeSequence");
- String ccPerDegreeFile = conf.get("ldbc.snb.bteronh.generator.ccPerDegree");
-
- BufferedReader reader = getFile(degreeSequenceFile,conf);
- ArrayList observedDegreeSequence = new ArrayList();
- String line;
- line = reader.readLine();
- while(line!=null) {
- observedDegreeSequence.add(Integer.parseInt(line));
- line = reader.readLine();
- }
-
- reader = getFile(ccPerDegreeFile,conf);
- ArrayList> ccPerDegree = new ArrayList>();
- line = reader.readLine();
- while(line!=null) {
- String [] splitLine = line.split(" ");
- ccPerDegree.add( new Pair(Long.parseLong(splitLine[0]), Double.parseDouble(splitLine[1])));
- line = reader.readLine();
- }
-
- System.out.println("Initializing BTER stats");
- BTERStats stats = new BTERStats();
- stats.initialize(numNodes, observedDegreeSequence, ccPerDegree, seed, (i) -> context.setStatus("Generated "+i+" degrees at mapper "+threadId));
-
- long totalWeight = stats.getWeightPhase1()+stats.getWeightPhase2();
- long blockSize = totalWeight / numThreads;
- if(threadId == 0) {
- long residual = totalWeight % numThreads;
- blockSize += residual;
- }
-
- System.out.println("Mapper "+threadId+": Generating "+blockSize+" edges out of "+totalWeight);
- Random random = new Random();
- random.setSeed(seed+threadId);
- for(long i = 0; i < blockSize; ++i) {
- Edge edge = Algorithms.BTERSample(stats,random);
- context.write(new LongWritable(edge.getTail()), new LongWritable(edge.getHead()));
- if( i % 100000 == 0 ) {
- context.setStatus("Generated "+i+" edges out of "+blockSize+" in mapper "+threadId);
- }
- }
- }
- }
-
-
- public static class HadoopBTERGeneratorReducer extends Reducer {
-
- @Override
- public void reduce(LongWritable tail, Iterable valueSet,
- Context context) throws IOException, InterruptedException {
- Set neighbors = new HashSet();
- for ( LongWritable head : valueSet ) {
- neighbors.add(head.get());
- }
-
- for( Long head : neighbors) {
- String str = new String(tail + " " + head + "\n");
- context.write(tail,new LongWritable(head));
- }
- }
- }
-
- private Configuration conf = null;
-
- public HadoopBTERGenerator(Configuration conf ) {
- this.conf = new Configuration(conf);
- }
-
- private static void writeToOutputFile(String filename, int numMaps, Configuration conf) {
- try {
- FileSystem dfs = FileSystem.get(conf);
- OutputStream output = dfs.create(new Path(filename));
- for (int i = 0; i < numMaps; i++)
- output.write((new String(i + "\n").getBytes()));
- output.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public void run() throws Exception {
-
- String workSpace = isHDFSPath(conf.get("ldbc.snb.bteronh.serializer.workspace"));
- if(workSpace == null) {
- throw new IOException("Ill-formed workspace URI. Workspace must start with hdfs://");
- }
- conf.set("ldbc.snb.bteronh.serializer.dataDir",workSpace+"/data");
- conf.set("ldbc.snb.bteronh.serializer.hadoopDir",workSpace+"/hadoop");
- String hadoopDir = new String( conf.get("ldbc.snb.bteronh.serializer.hadoopDir"));
- String dataDir = new String( conf.get("ldbc.snb.bteronh.serializer.dataDir"));
- String tempFile = hadoopDir+"/mrInputFile";
-
- String outputFileName = conf.get("ldbc.snb.bteronh.serializer.outputFileName");
- if(outputFileName != null){
- outputFileName = isHDFSPath(outputFileName);
- if(outputFileName == null) {
- throw new IOException("Ill-formed outputFileName URI"+outputFileName+". OutputFileName must start " +
- "with hdfs://");
- }
- } else {
- throw new IOException("You need to specify. ldbc.snb.bteronh.serializer.outputFileName");
- }
-
- System.out.println(dataDir);
- System.out.println(hadoopDir);
- System.out.println(outputFileName);
-
- FileSystem dfs = FileSystem.get(conf);
- dfs.delete(new Path(hadoopDir), true);
- dfs.delete(new Path(dataDir), true);
- dfs.delete(new Path(tempFile), true);
- writeToOutputFile(tempFile, Integer.parseInt(conf.get("ldbc.snb.bteronh.generator.numThreads")), conf);
-
- int numThreads = Integer.parseInt(conf.get("ldbc.snb.bteronh.generator.numThreads"));
- conf.setInt("mapreduce.input.lineinputformat.linespermap", 1);
- Job job = Job.getInstance(conf, "Generating Graph");
- job.setMapOutputKeyClass(LongWritable.class);
- job.setMapOutputValueClass(LongWritable.class);
- job.setOutputKeyClass(LongWritable.class);
- job.setOutputValueClass(LongWritable.class);
- job.setJarByClass(HadoopBTERGeneratorMapper.class);
- job.setMapperClass(HadoopBTERGeneratorMapper.class);
- job.setReducerClass(HadoopBTERGeneratorReducer.class);
- job.setNumReduceTasks(numThreads);
- job.setInputFormatClass(NLineInputFormat.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setPartitionerClass(HadoopEdgePartitioner.class);
- job.setOutputFormatClass(TextOutputFormat.class);
- FileInputFormat.setInputPaths(job, new Path(tempFile));
- FileOutputFormat.setOutputPath(job, new Path(outputFileName));
- if(!job.waitForCompletion(true)) {
- throw new Exception(job.toString());
- }
- }
-}
diff --git a/src/main/java/ldbc/snb/bteronh/structures/BTERStats.java b/src/main/java/ldbc/snb/bteronh/structures/BTERStats.java
deleted file mode 100644
index 7022b60..0000000
--- a/src/main/java/ldbc/snb/bteronh/structures/BTERStats.java
+++ /dev/null
@@ -1,253 +0,0 @@
-package ldbc.snb.bteronh.structures;
-
-import javafx.util.Pair;
-import ldbc.snb.bteronh.algorithms.Algorithms;
-import umontreal.iro.lecuyer.randvar.RandomVariateGen;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.function.Consumer;
-
-/**
- * Created by aprat on 14/07/16.
- */
-public class BTERStats {
-
- int maxDegree = Integer.MIN_VALUE;
-
- //Phase 1 data
- long [] i_g = null; //index of group in degree sequence
- long[] n_g = null; //number of buckets in group
- long[] b_g = null; //size of bucket in group
- double [] w_g = null; //weight of buckets in group
-
- //Phase 2 data
- long [] nfill_d = null;
- long [] nbulk_d = null;
- double [] wfill_d = null;
- double [] wbulk_d = null;
- double [] w_d = null;
- double [] r_d = null;
- long [] i_d = null;
- long [] n_d = null;
-
-
- double [] cumulativeGroups = null;
- double [] cumulativeDegrees = null;
- long weightPhase1;
- long weightPhase2;
-
- public void initialize(long numNodes, ArrayList observedDegreeSequence, ArrayList> observedCCPerDegree, int seed, Consumer continuation) {
-
- maxDegree = Integer.MIN_VALUE;
- RandomVariateGen randomVariateGen = Algorithms.GetDegreeSequenceSampler(observedDegreeSequence, numNodes, seed);
-
- System.out.println("Generating Degree Sequence");
- HashMap degrees = new HashMap();
- for(long i = 0; i < numNodes; ++i) {
- int degree = (int)randomVariateGen.nextDouble();
- degrees.compute(degree,(k,v)-> v == null ? 1 : v + 1 );
- maxDegree = degree > maxDegree ? degree : maxDegree;
- if(i % 1000000 == 0) {
- continuation.accept(i);
- }
- }
-
- double [] ccPerDegree = Algorithms.GenerateCCperDegree(observedCCPerDegree,maxDegree);
- initialize(numNodes, degrees,ccPerDegree);
- }
-
- public void initialize(long numNodes, HashMap degrees, double [] ccPerDegree) {
-
- if(maxDegree == Integer.MIN_VALUE) {
- maxDegree = Integer.MIN_VALUE;
- for (Integer key : degrees.keySet()) {
- maxDegree = key > maxDegree ? key : maxDegree;
- }
- }
-
- i_g = new long[maxDegree+1];
- b_g = new long[maxDegree+1];
- n_g = new long[maxDegree+1];
- w_g = new double[maxDegree+1];
- nfill_d = new long[maxDegree+1];
- nbulk_d = new long[maxDegree+1];
- wfill_d = new double[maxDegree+1];
- wbulk_d = new double[maxDegree+1];
- w_d = new double[maxDegree+1];
- r_d = new double[maxDegree+1];
- n_d = new long[maxDegree+1];
- i_d = new long[maxDegree+1];
- for(int i = 0; i < maxDegree+1; ++i) {
- i_g[i] = 0;
- b_g[i] = 0;
- n_g[i] = i+1;
- w_g[i] = 0;
-
- nfill_d[i] = 0;
- nbulk_d[i] = 0;
- wfill_d[i] = 0;
- wbulk_d[i] = 0;
-
- w_d[i] = 0;
- r_d[i] = 0;
- i_d[i] = 0;
- n_d[i] = 0;
- }
-
- // initializing n_d
- for(Map.Entry pair : degrees.entrySet()) {
- n_d[pair.getKey()] = (long)pair.getValue();
- }
-
-
- // initializing i_d
- i_d[2] = 0;
- for(int i = 3; i <= maxDegree; ++i) {
- i_d[i] = i_d[i-1] + n_d[i-1];
- }
- i_d[1] = i_d[maxDegree] + n_d[maxDegree];
- i_g[1] = i_d[maxDegree] + n_d[maxDegree];
-
- long [] n_dPrima = new long[maxDegree+1];
- int accum = 0;
- for(int i = 0; i <= maxDegree; ++i) {
- accum+=n_d[i];
- n_dPrima[i]=numNodes-accum;
- }
-
- //handling degree 1 nodes
- nfill_d[1] = 10*n_d[1];
- wfill_d[1] = 0.5*n_d[1];
- wbulk_d[1] = 0;
- w_d[1] = wbulk_d[1] + wfill_d[1];
- r_d[1] = 1;
-
- // Initializing group arrays.
- long nFillPrevious = 0;
- long g = -1;
- double dInternalPrevious = 0;
- for(int i = 2; i <= maxDegree; ++i) {
- if(nFillPrevious > 0) {
- nfill_d[i] = Math.min(nFillPrevious, n_d[i]);
- nFillPrevious-=nfill_d[i];
- wfill_d[i] = 0.5*nfill_d[i]*(i-dInternalPrevious);
- } else {
- nfill_d[i] = 0;
- wfill_d[i] = 0;
- }
- nbulk_d[i] = n_d[i] - nfill_d[i];
- if(nbulk_d[i] > 0) {
- g+=1;
- i_g[(int)g] = i_d[i] + nfill_d[i];
- b_g[(int)g] = (int)Math.ceil(nbulk_d[i]/(double)(i+1));
- n_g[(int)g] = i+1;
- if(b_g[(int)g]*(i+1) > (n_dPrima[i]+nbulk_d[i])) {
- n_g[(int)g] = (n_dPrima[i]+nbulk_d[i]);
- }
- double p = Math.pow(ccPerDegree[i],1/3.0);
- dInternalPrevious = (n_g[(int)g]-1)*p;
- wbulk_d[i] = 0.5*nbulk_d[i]*(i-dInternalPrevious);
- w_g[(int)g] = b_g[(int)g]*0.5*n_g[(int)g]*(n_g[(int)g]-1)*Math.log(1/(1.0-p));
- nFillPrevious = (b_g[(int)g]*n_g[(int)g]) - nbulk_d[i];
- } else {
- wbulk_d[i] = 0;
- }
- w_d[i] = wfill_d[i] + wbulk_d[i];
- r_d[i] = wfill_d[i] / w_d[i];
- }
-
- long [] newi_g = new long[(int)g+1];
- long [] newn_g = new long[(int)g+1]; //number of buckets in group
- long [] newb_g = new long[(int)g+1]; //size of bucket in group
- double [] neww_g = new double[(int)g+1]; //weight of buckets in group
- for(int i = 0; i < g+1; ++i) {
- newi_g[i] = i_g[i];
- newn_g[i] = n_g[i];
- newb_g[i] = b_g[i];
- neww_g[i] = w_g[i];
- }
-
- i_g = newi_g;
- n_g = newn_g;
- b_g = newb_g;
- w_g = neww_g;
-
- weightPhase1 = 0L;
- weightPhase2 = 0L;
- for(int i = 0; i < getNumGroups(); ++i) {
- weightPhase1+= getGroupWeight(i);
- }
-
- for(int i = 0; i < (getMaxDegree()+1); ++i) {
- weightPhase2+=getDegreeWeight(i);
- }
-
- cumulativeGroups = new double[getNumGroups()];
- cumulativeGroups[0] = getGroupWeight(0) / (double)weightPhase1;
- for(int i = 1; i < getNumGroups(); ++i) {
- cumulativeGroups[i] = Math.min(cumulativeGroups[i-1] + getGroupWeight(i) / (double)weightPhase1, 1.0);
- }
-
- cumulativeDegrees = new double[getMaxDegree()+1];
- cumulativeDegrees[0] = getDegreeWeight(0) / (double)weightPhase2;
- for(int i = 1; i < (getMaxDegree()+1); ++i) {
- cumulativeDegrees[i] = Math.min(cumulativeDegrees[i-1] + getDegreeWeight(i) / (double)weightPhase2,1.0);
- }
- }
-
- public long getGroupIndex(int group) {return i_g[group];}
-
- public long getGroupBucketSize(int group) {return n_g[group];}
-
- public long getGroupNumBuckets(int group) {return b_g[group];}
-
- public double getGroupWeight(int group) {return w_g[group];}
-
- public double [] getGroupWeights() { return w_g;}
-
- public long getDegreeIndex(int degree) { return i_d[degree];}
-
- public long getDegreeNFill(int degree) { return nfill_d[degree];}
-
- public long getDegreeNBulk(int degree) { return nbulk_d[degree];}
-
- public double getDegreeWFill(int degree) { return wfill_d[degree];}
-
- public double getDegreeWBulk(int degree) { return wbulk_d[degree];}
-
- public double getDegreeWeight(int degree) { return w_d[degree];}
-
- public double [] getDegreeWeights() { return w_d;}
-
- public double getDegreeWeightRatio( int degree ) { return r_d[degree];}
-
- public double [] getDegreeWeightRatios() { return r_d;}
-
- public double getDegreeN(int degree) { return n_d[degree];}
-
- public int getNumGroups() {
- return n_g.length;
- }
-
- public int getMaxDegree() { return maxDegree;}
-
- public double[] getCumulativeGroups() {
- return cumulativeGroups;
- }
-
- public double[] getCumulativeDegrees() {
- return cumulativeDegrees;
- }
-
- public long getWeightPhase1() {
- return weightPhase1;
- }
-
- public long getWeightPhase2() {
- return weightPhase2;
- }
-
-}
diff --git a/src/main/java/ldbc/snb/bteronhplus/Main.java b/src/main/java/ldbc/snb/bteronhplus/Main.java
new file mode 100644
index 0000000..9b55efa
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/Main.java
@@ -0,0 +1,240 @@
+package ldbc.snb.bteronhplus;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import ldbc.snb.bteronhplus.algorithms.Partitioning;
+import ldbc.snb.bteronhplus.structures.*;
+import org.jgrapht.Graph;
+import org.jgrapht.alg.ConnectivityInspector;
+import org.jgrapht.graph.DefaultEdge;
+import org.jgrapht.graph.SimpleGraph;
+import org.jgrapht.graph.builder.GraphBuilder;
+
+import java.io.FileWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.*;
+
+
+public class Main {
+
+ public static class Arguments {
+
+ @Parameter(names = {"-d", "--degrees"}, description = "The file with the degrees", required = true)
+ public String degreesFile;
+
+ @Parameter(names = {"-cc", "--clustering"}, description = "The file with the clustering coefficient " +
+ "distributions per degree", required = true)
+ public String ccsFile;
+
+ @Parameter(names = {"-c", "--communities"}, description = "The file with the community sizes", required = true)
+ public String communitiesfile;
+
+ @Parameter(names = {"-o", "--output"}, description = "The output file", required = true)
+ public String outputFileName;
+
+ @Parameter(names = {"-p", "--density"}, description = "The community densities file", required = true)
+ public String densityFileName;
+
+ @Parameter(names = {"-s", "--size"}, description = "The size of the graph", required = true)
+ public int graphSize;
+
+ @Parameter(names = {"-cs", "--csprefix"}, description = "The community structure file name prefix", required =
+ true)
+ public String communityStructureFileNamePrefix;
+
+ @Parameter(names = {"-l", "--levels"}, description = "The number of community structure levels " )
+ public int levels = 2;
+
+ @Parameter(names = {"-m", "--modules"}, description = "The modules file prefix", required = true)
+ public String modulesPrefix;
+
+
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ Arguments arguments = new Arguments();
+ new JCommander(arguments, args);
+
+ System.out.println("Degree file: "+arguments.degreesFile);
+ System.out.println("CCS file: "+arguments.ccsFile);
+ System.out.println("Communities file: "+arguments.communitiesfile);
+ System.out.println("Density file: "+arguments.densityFileName);
+ System.out.println("Modules prefix: "+arguments.modulesPrefix);
+
+ GraphStats graphStats = new GraphStats(arguments.degreesFile,
+ arguments.ccsFile,
+ arguments.communitiesfile,
+ arguments.densityFileName);
+
+
+ byte[] byteArray = Files.readAllBytes(Paths.get(arguments.communityStructureFileNamePrefix+1));
+ String blockModelData = new String(byteArray);
+ byteArray = Files.readAllBytes(Paths.get(arguments.communityStructureFileNamePrefix+1+".children"));
+ String childrenData = new String(byteArray);
+ BlockModel blockModel = new BlockModel(blockModelData, childrenData);
+
+ Random random = new Random();
+ random.setSeed(12345L);
+
+ //BasicCommunityStreamer communityStreamer = new BasicCommunityStreamer(graphStats);
+ //CorePeripheryCommunityStreamer communityStreamer = new CorePeripheryCommunityStreamer(graphStats,random);
+ RealCommunityStreamer communityStreamer = new RealCommunityStreamer(graphStats, arguments.modulesPrefix+
+ "communities", random);
+ List> partition = Partitioning.partition(random, blockModel,
+ communityStreamer,
+ arguments.graphSize);
+
+ // Counting total number of nodes
+ long totalDegree = 0L;
+ long totalExternalDegree = 0L;
+ long totalNumNodes = 0L;
+ long internalDegreePerBlock[] = new long[partition.size()];
+ long totalDegreePerBlock[] = new long[partition.size()];
+ Arrays.fill(internalDegreePerBlock, 0L);
+ Arrays.fill(totalDegreePerBlock, 0L);
+
+ int index = 0;
+ for(Map counts : partition) {
+ for(Map.Entry entry : counts.entrySet()) {
+ Community model = communityStreamer.getModel(entry.getKey());
+ totalNumNodes += entry.getValue()*model.getSize();
+ totalDegree += entry.getValue()*(model.getExternalDegree()+model.getInternalDegree());
+ totalExternalDegree += entry.getValue()*(model.getExternalDegree());
+ internalDegreePerBlock[index] += entry.getValue()*model.getInternalDegree();
+ totalDegreePerBlock[index] += entry.getValue()*(model.getInternalDegree()+model.getExternalDegree());
+ }
+ index++;
+ }
+
+ System.out.println("Generating community edges");
+ FileWriter outputFile = new FileWriter(arguments.outputFileName);
+
+ // Computing offsets and generating internal community edges
+ long offsets[] = new long[partition.size()];
+
+ offsets[0] = 0;
+ for(int i = 1; i < offsets.length; ++i) {
+ Map counts = partition.get(i-1);
+ long blockSize = 0;
+ for(Map.Entry entry : counts.entrySet()) {
+ Community model = communityStreamer.getModel(entry.getKey());
+ blockSize += entry.getValue()*model.getSize();
+ }
+ offsets[i] = offsets[i-1] + blockSize;
+ }
+
+ List blockSamplers = new ArrayList();
+
+ for(Map entry : partition ) {
+ blockSamplers.add(new BlockSampler(entry, communityStreamer));
+ }
+
+
+ System.out.println("Generating external edges");
+ System.out.println("Total number of external edges to generate: "+(totalExternalDegree/2));
+ long totalExternalGeneratedEdges = 0;
+ long totalExpectedGeneratedEdges = 0;
+ double sumDensities = 0.0;
+ long totalBrokenBlocks = 0L;
+ long totalNumEdgesBelowThreshold = 0L;
+ long totalNumEdgesNegative = 0L;
+ long numCommunitiesIfWellConnected = 0L;
+ long numExcess = 0;
+ for(BlockModel.ModelEntry entry : blockModel.getEntries().values()) {
+ for(Map.Entry neighbor : entry.degree.entrySet()) {
+ sumDensities += neighbor.getValue();
+ BlockSampler sampler = blockSamplers.get(entry.id);
+ GraphBuilder builder = SimpleGraph.createBuilder(DefaultEdge.class);
+
+ double expectedDegree = entry.totalDegree * totalDegree;
+ if(entry.id == neighbor.getKey()) {
+
+
+ long numEdges = (Math.round(neighbor.getValue() * totalDegree) - internalDegreePerBlock[entry
+ .id])/2;
+
+
+ if(numEdges < (sampler.getNumCommunities()-1) && sampler.getNumCommunities() > 1){
+ totalNumEdgesBelowThreshold++;
+ if(numEdges < 0) totalNumEdgesNegative++;
+ } else {
+ numCommunitiesIfWellConnected++;
+ }
+
+ sampler.generateCommunityEdges(outputFile,
+ partition.get(entry.id),
+ communityStreamer,
+ offsets[entry.id],
+ builder);
+
+ long darwiniEdges = sampler.darwini(outputFile,random,offsets[entry.id]);
+ totalExternalGeneratedEdges+=darwiniEdges;
+ numEdges-=darwiniEdges;
+
+ sampler.generateConnectedGraph(outputFile, random, offsets[entry.id], builder);
+
+ numEdges -= sampler.getNumCommunities() - 1;
+
+ for(int i = 0; i < numEdges; ++i) {
+ long node1 = sampler.sample(random, offsets[entry.id]);
+ long node2 = sampler.sample(random, offsets[entry.id]);
+ totalExpectedGeneratedEdges++;
+ if (node1 != -1 && node2 != -1) {
+ outputFile.write(node1 + "\t" + node2 + "\n");
+ totalExternalGeneratedEdges++;
+
+ if(node1 != node2) {
+ builder.addEdge(node1, node2);
+ }
+ }
+ }
+
+ Graph graph = builder.build();
+ ConnectivityInspector connectivityInspector = new ConnectivityInspector<>(graph);
+ List> connectedComponents = connectivityInspector.connectedSets();
+ if (connectedComponents.size() > 1) {
+ totalBrokenBlocks++;
+ }
+ }
+ }
+ }
+
+ for(BlockModel.ModelEntry entry : blockModel.getEntries().values()) {
+ for (Map.Entry neighbor : entry.degree.entrySet()) {
+
+
+ if(entry.id < neighbor.getKey()) {
+ BlockSampler sampler1 = blockSamplers.get(entry.id);
+ BlockSampler sampler2 = blockSamplers.get(neighbor.getKey());
+ long numEdges= Math.round(neighbor.getValue() * totalDegree);
+
+ for(int i = 0; i < numEdges; ++i) {
+ long node1 = sampler1.sample(random, offsets[entry.id]);
+ long node2 = sampler2.sample(random, offsets[neighbor.getKey()]);
+ totalExpectedGeneratedEdges++;
+ if (node1 != -1 && node2 != -1) {
+ outputFile.write(node1 + "\t" + node2 + "\n");
+ totalExternalGeneratedEdges++;
+ }
+ }
+
+ }
+
+ }
+ }
+
+
+ System.out.println("Total number of external edges generated: "+totalExternalGeneratedEdges);
+ System.out.println("Total expected generated: "+totalExpectedGeneratedEdges);
+ System.out.println("Total sum densities: "+sumDensities);
+ System.out.println("Total broken blocks: "+totalBrokenBlocks);
+ System.out.println("Total num edges below threshold: "+totalNumEdgesBelowThreshold);
+ System.out.println("Total num edges negative: "+totalNumEdgesNegative);
+ System.out.println("Total num communities if well connected: "+numCommunitiesIfWellConnected);
+
+ outputFile.close();
+
+ }
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/algorithms/Generator.java b/src/main/java/ldbc/snb/bteronhplus/algorithms/Generator.java
new file mode 100644
index 0000000..d1144b1
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/algorithms/Generator.java
@@ -0,0 +1,14 @@
+package ldbc.snb.bteronhplus.algorithms;
+
+import ldbc.snb.bteronhplus.structures.BlockSampler;
+import ldbc.snb.bteronhplus.structures.Community;
+import ldbc.snb.bteronhplus.structures.RealCommunityStreamer;
+
+import java.io.FileWriter;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+public class Generator {
+
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/algorithms/Partitioning.java b/src/main/java/ldbc/snb/bteronhplus/algorithms/Partitioning.java
new file mode 100644
index 0000000..082a70f
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/algorithms/Partitioning.java
@@ -0,0 +1,448 @@
+package ldbc.snb.bteronhplus.algorithms;
+
+import ldbc.snb.bteronhplus.structures.BlockModel;
+import ldbc.snb.bteronhplus.structures.SuperNode;
+import ldbc.snb.bteronhplus.structures.CommunityStreamer;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
+
+public class Partitioning {
+
+ private static double computeFactor(BlockModel.ModelEntry entry,
+ double totalBlockDegree,
+ double sumInternalDegree,
+ long numCommunities
+ ) {
+
+ double ratio = entry.externalDegree / entry.totalDegree;
+ double externalDegree = totalBlockDegree * ratio;
+ double internalDegree = totalBlockDegree - externalDegree;
+ double interDegree = internalDegree - (sumInternalDegree);
+
+ double minimum = computeMinimumInterDegree((numCommunities), 0.05);
+ if(interDegree >= minimum) {
+ return 1;
+ }
+ //return 1 / (minimum - interDegree+1);
+ return 1.0;
+ }
+
+ private static double computeMinimumInterDegree(long size, double error) {
+ /*double prob = ((1.0 + error)*Math.log(size) / (double)(size) + 1)+0.05;
+ return ((prob*(size*(size-1) / 2.0)))*2.0;*/
+ return size-1;
+ }
+
+ private static double computeRatio(double totalDegree, double internalDegree, double expectedRatio) {
+
+ return expectedRatio;
+
+ }
+
+ private static double combineScores(double sizeScore,
+ double internalDegreeScore,
+ double externalDegreeScore) {
+ return (/*sizeScore + */internalDegreeScore + externalDegreeScore) ;
+ }
+
+ private static double computeScore(long totalSize,
+ long totalDegree,
+ long totalInternalDegree,
+ long totalExternalDegree,
+ BlockModel.ModelEntry entry,
+ double size,
+ double internalDegree,
+ double externalDegree) {
+
+ double expectedSize = (entry.size*totalSize);
+ double expectedInternalDegree = (entry.totalDegree - entry.externalDegree)*totalDegree;
+ double expectedExternalDegree = (entry.externalDegree)*totalDegree;
+
+
+ /*double normalizeFactorEx = totalInternalDegree/(double)totalExternalDegree;
+ double normalizeFactorSize = totalInternalDegree/(double)totalSize;*/
+
+ double sizeScore = Math.pow((expectedSize-size),2);
+ double internalDegreeScore = Math.pow((expectedInternalDegree-internalDegree), 2);
+ double externalDegreeScore = Math.pow((expectedExternalDegree-externalDegree),2);
+ return combineScores(sizeScore, internalDegreeScore, externalDegreeScore);
+
+ }
+
+ private static double initializePartialScores(BlockModel blockModel,
+ double partialScores[],
+ long totalSize,
+ long totalDegree,
+ long totalInternalDegree,
+ long totalExternalDegree,
+ long currentBlockSize[],
+ long currentBlockInternalDegree[],
+ long currentBlockExternalDegree[],
+ double currentBlockRatio[]
+ ) {
+
+ int numBlocks = blockModel.getNumBlocks();
+ double score = 0.0;
+ for(int i = 0; i < numBlocks; ++i) {
+ BlockModel.ModelEntry entry = blockModel.getEntries().get(i);
+ double blockSize = currentBlockSize[i];
+ double blockTotalDegree = currentBlockInternalDegree[i] + currentBlockExternalDegree[i];
+ double blockRatio = currentBlockRatio[i];
+ double blockInternalDegree = (blockTotalDegree*(1.0-blockRatio));
+ double blockExternalDegree = (blockTotalDegree*blockRatio);
+
+ partialScores[i] = computeScore(totalSize,
+ totalDegree,
+ totalInternalDegree,
+ totalExternalDegree,
+ entry,
+ blockSize,
+ blockInternalDegree,
+ blockExternalDegree);
+ score += partialScores[i];
+ }
+ return score;
+ }
+
+
+ private static double score(BlockModel blockModel,
+ double currentScore,
+ long totalSize,
+ long totalDegree,
+ long totalInternalDegree,
+ long totalExternalDegree,
+ long currentBlockSize[],
+ long currentBlockInternalDegree[],
+ long currentBlockExternalDegree[],
+ double currentBlockRatio[],
+ long size,
+ long internalDegree,
+ long externalDegree,
+ int block) {
+
+
+ double score = currentScore;
+
+ BlockModel.ModelEntry entry = blockModel.getEntries().get(block);
+ double blockSize = currentBlockSize[block];
+ double blockTotalDegree = currentBlockInternalDegree[block] + currentBlockExternalDegree[block];
+ double blockRatio = currentBlockRatio[block];
+ double blockInternalDegree = (blockTotalDegree*(1.0-blockRatio));
+ double blockExternalDegree = (blockTotalDegree*blockRatio);
+
+ double expectedRatio = entry.externalDegree / entry.totalDegree;
+ blockSize+=size;
+ blockTotalDegree += internalDegree;
+ blockTotalDegree += externalDegree;
+ blockRatio = computeRatio(blockTotalDegree, blockInternalDegree, expectedRatio);
+
+ blockInternalDegree = (blockTotalDegree*(1.0-blockRatio));
+ blockExternalDegree = (blockTotalDegree*(blockRatio));
+
+
+ score += computeScore(totalSize,
+ totalDegree,
+ totalInternalDegree,
+ totalExternalDegree,
+ entry,
+ blockSize,
+ blockInternalDegree,
+ blockExternalDegree);
+ return score;
+ }
+
+
+ public static List> partition(Random random, BlockModel blockModel,
+ CommunityStreamer streamer,
+ long numNodes) {
+
+ List> superNodesPerBlock = new ArrayList>();
+ int numBlocks = blockModel.getNumBlocks();
+ for(int i = 0; i < numBlocks; ++i) {
+ superNodesPerBlock.add(new HashMap());
+ }
+
+ long currentBlockSize[] = new long[numBlocks];
+ Arrays.fill(currentBlockSize, 0L);
+ long currentBlockInternalDegree[] = new long[numBlocks];
+ Arrays.fill(currentBlockInternalDegree, 0L);
+ long currentBlockExternalDegree[] = new long[numBlocks];
+ Arrays.fill(currentBlockExternalDegree, 0L);
+ double currentBlockRatio[] = new double[numBlocks];
+ Arrays.fill(currentBlockRatio, 0.0);
+ long currentBlockInterBudgetDegree[] = new long[numBlocks];
+ Arrays.fill(currentBlockInterBudgetDegree, 0L);
+ long currentBlockNumCommunities[] = new long[numBlocks];
+ Arrays.fill(currentBlockNumCommunities, 0L);
+
+ double partialScores[] = new double[numBlocks];
+ Arrays.fill(partialScores,0.0);
+
+ long totalNodes = 0L;
+ long totalDegree = 0L;
+
+ List superNodes = new ArrayList();
+ while( totalNodes < numNodes) {
+ SuperNode nextSuperNode = streamer.next();
+ superNodes.add(nextSuperNode);
+ totalNodes+=nextSuperNode.getSize();
+ totalDegree+=nextSuperNode.getInternalDegree();
+ totalDegree+=nextSuperNode.getExternalDegree();
+ }
+
+ long totalInternalDegree = 0L;
+ long totalExternalDegree = 0L;
+ for(BlockModel.ModelEntry entry : blockModel.getEntries().values()) {
+ totalInternalDegree += (long)(totalDegree*(entry.totalDegree - entry.externalDegree));
+ totalExternalDegree += (long)(totalDegree*(entry.externalDegree));
+ }
+
+ superNodes.sort(new Comparator() {
+ @Override
+ public int compare(SuperNode superNode1, SuperNode superNode2) {
+ if(superNode1.getSize() > superNode2.getSize()) return -1;
+ if(superNode1.getSize() == superNode2.getSize()) return 0;
+ return 1;
+ }
+ });
+
+ int count = 1;
+
+ long totalObservedNodes = 0L;
+ long totalObservedDegree = 0L;
+
+ for( SuperNode nextSuperNode : superNodes) {
+
+ int bestBlock = random.nextInt(blockModel.getNumBlocks());
+ double currentScore = initializePartialScores(blockModel,
+ partialScores,
+ totalNodes,
+ totalDegree,
+ totalInternalDegree,
+ totalExternalDegree,
+ currentBlockSize,
+ currentBlockInternalDegree,
+ currentBlockExternalDegree,
+ currentBlockRatio);
+
+ double nextScore = score(blockModel,
+ currentScore - partialScores[bestBlock],
+ totalNodes,
+ totalDegree,
+ totalInternalDegree,
+ totalExternalDegree,
+ currentBlockSize,
+ currentBlockInternalDegree,
+ currentBlockExternalDegree,
+ currentBlockRatio,
+ nextSuperNode.getSize(),
+ nextSuperNode.getInternalDegree(),
+ nextSuperNode.getExternalDegree(),
+ bestBlock);
+
+ double bestImprovement = currentScore - nextScore;
+ double blockTotalDegree = currentBlockInternalDegree[bestBlock] +
+ currentBlockExternalDegree[bestBlock] + nextSuperNode
+ .getInternalDegree() + nextSuperNode.getExternalDegree();
+
+ /*double factor = computeFactor(blockModel.getEntries().get(bestBlock),
+ blockTotalDegree,
+ currentBlockInternalDegree[bestBlock] + nextSuperNode.getInternalDegree(),
+ currentBlockNumCommunities[bestBlock] + 1 );*/
+
+ double factor = (1.0 - (currentBlockSize[bestBlock] + nextSuperNode.getSize())/(double)(blockModel
+ .getEntries().get(bestBlock).size*totalNodes));
+ factor = 1.0;
+
+ if(bestImprovement < 0.0) {
+ bestImprovement *= 1.0/factor;
+ } else {
+ bestImprovement *= factor;
+ }
+
+ for(int j = 0; j < numBlocks; ++j) {
+
+ BlockModel.ModelEntry entry = blockModel.getEntries().get(j);
+ nextScore = score(blockModel,
+ currentScore - partialScores[j],
+ totalNodes,
+ totalDegree,
+ totalInternalDegree,
+ totalExternalDegree,
+ currentBlockSize,
+ currentBlockInternalDegree,
+ currentBlockExternalDegree,
+ currentBlockRatio,
+ nextSuperNode.getSize(),
+ nextSuperNode.getInternalDegree(),
+ nextSuperNode.getExternalDegree(),
+ j);
+
+
+ double improvement = currentScore - nextScore;
+
+
+ blockTotalDegree = currentBlockInternalDegree[j] +
+ currentBlockExternalDegree[j] + nextSuperNode
+ .getInternalDegree() + nextSuperNode.getExternalDegree();
+
+ if(improvement < 0.0) {
+ improvement *= 1.0/factor;
+ } else {
+ improvement *= factor;
+ }
+
+ /*factor = computeFactor(entry,
+ blockTotalDegree,
+ currentBlockInternalDegree[bestBlock] + nextSuperNode.getInternalDegree(),
+ currentBlockNumCommunities[bestBlock] + 1 );*/
+
+ factor = (1.0 - (currentBlockSize[j] + nextSuperNode.getSize())/(double)(entry
+ .size*totalNodes));
+ factor = 1.0;
+
+ if(improvement < 0.0) {
+ improvement *= (1.0/factor);
+ } else {
+ improvement *= factor;
+ }
+
+ /*if(!meetsInterDegree(entry,
+ blockTotalDegree,
+ currentBlockInternalDegree[j] + nextSuperNode.getInternalDegree(),
+ currentBlockNumCommunities[j] + 1 )) {
+ improvement = -Double.MAX_VALUE;
+
+ }*/
+
+
+ if(improvement > bestImprovement) {
+ bestBlock = j;
+ bestImprovement = improvement;
+ }
+ }
+
+ currentBlockSize[bestBlock]+=nextSuperNode.getSize();
+ currentBlockInternalDegree[bestBlock]+=nextSuperNode.getInternalDegree();
+ currentBlockExternalDegree[bestBlock]+=nextSuperNode.getExternalDegree();
+ long totalBlockDegree = currentBlockInternalDegree[bestBlock] + currentBlockExternalDegree[bestBlock];
+ BlockModel.ModelEntry entry = blockModel.getEntries().get(bestBlock);
+ double expectedRatio = entry.externalDegree / entry.totalDegree;
+
+ currentBlockRatio[bestBlock] = computeRatio((long)totalBlockDegree,
+ currentBlockInternalDegree[bestBlock],
+ expectedRatio);
+
+ currentBlockInterBudgetDegree[bestBlock] = Math.abs(currentBlockInterBudgetDegree[bestBlock] -
+ nextSuperNode.getExternalDegree());
+
+ currentBlockNumCommunities[bestBlock]++;
+
+
+ superNodesPerBlock.get(bestBlock).merge(nextSuperNode.getId(), 1L, Long::sum);
+
+ totalObservedDegree+=nextSuperNode.getInternalDegree()+nextSuperNode.getExternalDegree();
+ totalObservedNodes+=nextSuperNode.getSize();
+
+
+ if( count % 1000 == 0) {
+ System.out.println("Distributed "+count+" superNodes amounting "+totalObservedNodes+" out of " +
+ numNodes);
+ }
+ count++;
+ }
+
+ printStats(blockModel, currentBlockSize, currentBlockInternalDegree, currentBlockExternalDegree, currentBlockRatio);
+
+ System.out.println("Number of nodes observed "+totalObservedNodes);
+ System.out.println("Number of degree observed "+totalObservedDegree);
+ System.out.println("Number of super nodes consumed "+count);
+ return superNodesPerBlock;
+ }
+
+ private static void printStats(BlockModel blockModel,
+ long currentBlockSize[],
+ long currentBlockInternalDegree[],
+ long currentBlockExternalDegree[],
+ double currentBlockRatio[]
+ ) {
+
+ long totalDegree = 0;
+ for(int i = 0; i < currentBlockInternalDegree.length; ++i) {
+ totalDegree+=currentBlockInternalDegree[i];
+ totalDegree+=currentBlockExternalDegree[i];
+ }
+
+ long totalSize = 0;
+ for(int i = 0; i < currentBlockSize.length; ++i) {
+ totalSize += currentBlockSize[i];
+ }
+
+ long totalExpectedInternalDegree = 0;
+ long totalExpectedExternalDegree = 0;
+
+ long totalObservedInternalDegree = 0;
+ long totalObservedExternalDegree = 0;
+
+ double sizeScore = 0.0;
+ double internalDegreeScore = 0.0;
+ double externalDegreeScore = 0.0;
+
+ try {
+ FileWriter partitioningSizes = new FileWriter(new File("partitioning.sizes"));
+ for(BlockModel.ModelEntry entry : blockModel.getEntries().values()) {
+ long expectedSize = (long)(entry.size*totalSize);
+ long observedSize = (long)(currentBlockSize[(int)entry.id]);
+ partitioningSizes.write(entry.id+" "+expectedSize+" "+observedSize+"\n");
+ sizeScore += Math.pow((expectedSize-observedSize), 2 );
+ }
+ partitioningSizes.close();
+
+ FileWriter partitioningInternalDegree = new FileWriter(new File("partitioning.internalDegree"));
+ for(BlockModel.ModelEntry entry : blockModel.getEntries().values()) {
+ long expectedInternalDegree = (long)((entry.totalDegree - entry.externalDegree)*totalDegree);
+ totalExpectedInternalDegree += expectedInternalDegree;
+ long observedInternalDegree = (long)((currentBlockInternalDegree[(int)entry.id] +
+ currentBlockExternalDegree[(int)(entry.id)]) * (1-currentBlockRatio[(int)entry.id]));
+ totalObservedInternalDegree += observedInternalDegree;
+
+ internalDegreeScore += Math.pow((expectedInternalDegree - observedInternalDegree) ,2);
+
+ partitioningInternalDegree.write(entry.id+" "+expectedInternalDegree+" " +
+ ""+observedInternalDegree+"\n");
+ }
+ partitioningInternalDegree.close();
+
+ FileWriter partitioningExternalDegree = new FileWriter(new File("partitioning.externalDegree"));
+ for(BlockModel.ModelEntry entry : blockModel.getEntries().values()) {
+ long expectedExternalDegree = (long)(entry.externalDegree*totalDegree);
+ totalExpectedExternalDegree += expectedExternalDegree;
+ long observedExternalDegree = (long)((currentBlockInternalDegree[(int)entry.id] +
+ currentBlockExternalDegree[(int)(entry.id)]) * (currentBlockRatio[(int)entry.id]));
+
+ totalObservedExternalDegree += observedExternalDegree;
+
+ externalDegreeScore += Math.pow((expectedExternalDegree - observedExternalDegree),2);
+
+ partitioningExternalDegree.write(entry.id+" "+expectedExternalDegree+" " +
+ observedExternalDegree+"\n");
+ }
+ partitioningExternalDegree.close();
+ } catch( IOException e) {
+ e.printStackTrace();
+ }
+
+ System.out.println("Expected vs Observed internal degree: "+totalExpectedInternalDegree+" "+totalObservedInternalDegree);
+ System.out.println("Expected vs Observed external degree: "+totalExpectedExternalDegree+" " +
+ ""+totalObservedExternalDegree);
+
+ System.out.println("Size Score: "+(long)sizeScore);
+ System.out.println("Internal Degree Score: "+(long)internalDegreeScore);
+ System.out.println("External Degree Score: "+(long)externalDegreeScore);
+
+ }
+
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/algorithms/SuperNodeUtils.java b/src/main/java/ldbc/snb/bteronhplus/algorithms/SuperNodeUtils.java
new file mode 100644
index 0000000..add41f7
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/algorithms/SuperNodeUtils.java
@@ -0,0 +1,90 @@
+package ldbc.snb.bteronhplus.algorithms;
+
+import ldbc.snb.bteronhplus.structures.BlockModel;
+import ldbc.snb.bteronhplus.structures.SuperNode;
+import ldbc.snb.bteronhplus.structures.CommunityCluster;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class SuperNodeUtils {
+
+ /*private static CommunityCluster buildSuperNodeClusterHierarchy(List blockModelHierarchy,
+ Map clusters,
+ int id,
+ List children,
+ int level
+ ) {
+ if(level == blockModelHierarchy.size()) {
+ return clusters.get(id);
+
+ } else {
+
+ BlockModel blockModel = blockModelHierarchy.get(level);
+
+ List childrenClusters = new ArrayList();
+ for(Integer child : children ) {
+ List subchildren = blockModel.getChildren().get(child);
+ CommunityCluster cluster = buildSuperNodeClusterHierarchy(blockModelHierarchy,
+ clusters,
+ child,
+ subchildren,
+ level+1);
+ if(cluster != null) {
+ childrenClusters.add(cluster);
+ }
+ }
+
+ if(childrenClusters.size() == 0) {
+ return null;
+ }
+
+ double internalDegree = 0;
+ double totalDegree = 0;
+
+ for(SuperNode child1 : childrenClusters) {
+ BlockModel.ModelEntry entry = blockModel.getEntries().get(child1.getId());
+ totalDegree += entry.totalDegree;
+ for(SuperNode child2 : childrenClusters) {
+ Double degree = entry.degree.get(child2.getId());
+ if(degree != null) {
+ internalDegree += degree;
+ }
+ }
+ }
+
+ double externalRatio = 1.0 - internalDegree / (totalDegree);
+
+ return new CommunityCluster(id, childrenClusters, externalRatio);
+
+ }
+
+ }
+
+ public static CommunityCluster buildSuperNodeClusterHierarchy(List blockModelHierarchy,
+ Map clusters) {
+
+
+ BlockModel root = blockModelHierarchy.get(0);
+
+ List children = new ArrayList();
+ for(Map.Entry> entry : root.getChildren().entrySet() ) {
+ CommunityCluster cluster = buildSuperNodeClusterHierarchy(blockModelHierarchy,
+ clusters,
+ entry.getKey(),
+ entry.getValue(),
+ 1);
+ if(cluster != null) {
+ children.add(cluster);
+ }
+ }
+
+ if(children.size() == 0) {
+ return null;
+ }
+
+ return new CommunityCluster(0, children, 0.0);
+ }
+ */
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/hadoop/HadoopBTERGenerator.java b/src/main/java/ldbc/snb/bteronhplus/hadoop/HadoopBTERGenerator.java
new file mode 100644
index 0000000..5ecbbc9
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/hadoop/HadoopBTERGenerator.java
@@ -0,0 +1,83 @@
+package ldbc.snb.bteronhplus.hadoop;
+
+import ldbc.snb.bteronhplus.structures.Edge;
+import ldbc.snb.bteronhplus.tools.FileTools;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+/**
+ * Created by aprat on 16/08/16.
+ */
+public class HadoopBTERGenerator {
+
+
+ public static class HadoopBTERGeneratorMapper extends Mapper {
+
+
+ @Override
+ public void map(LongWritable key, Text value, Context context)
+ throws IOException, InterruptedException {
+
+ /* int threadId = Integer.parseInt(value.toString());
+
+ Configuration conf = context.getConfiguration();
+ String bterStatsFile = conf.get("ldbc.snb.bteronh.generator.bterstats");
+ int numThreads = conf.getInt("ldbc.snb.bteronh.generator.numThreads",1);
+ int seed = conf.getInt("ldbc.snb.bteronh.generator.seed",0);
+
+ BTERStats stats = FileTools.deserializeObject(bterStatsFile,conf);
+
+ long offset = conf.getLong("offset",0);
+ int communityId = conf.getInt("communityId",0);
+
+ long totalWeight = stats.getWeightPhase1()+stats.getWeightPhase2();
+ long blockSize = totalWeight / numThreads;
+ if(threadId == 0) {
+ long residual = totalWeight % numThreads;
+ blockSize += residual;
+ }
+ System.out.println("Edges to generate at reducer "+threadId+" for parition "+communityId+":"+blockSize);
+
+ System.out.println("Mapper "+threadId+": Generating "+blockSize+" edges out of "+totalWeight);
+ Random random = new Random();
+ random.setSeed(seed+threadId);
+ for(long i = 0; i < blockSize; ++i) {
+ Edge edge = BTER.BTERSample(stats,random,offset);
+ context.write(new LongWritable(edge.getTail()), new LongWritable(edge.getHead()));
+ if( i % 100000 == 0 ) {
+ context.setStatus("Generated "+i+" edges out of "+blockSize+" in mapper "+threadId);
+ }
+ }
+ */
+ }
+ }
+
+
+ public static class HadoopBTERGeneratorReducer extends Reducer {
+
+ @Override
+ public void reduce(LongWritable tail, Iterable valueSet,
+ Context context) throws IOException, InterruptedException {
+ /* Set neighbors = new HashSet();
+ for ( LongWritable head : valueSet ) {
+ neighbors.add(head.get());
+ }
+
+ for( Long head : neighbors) {
+ //String str = new String(tail + " " + head + "\n");
+ context.write(tail,new LongWritable(head));
+ }
+ */
+ }
+
+ }
+
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/hadoop/HadoopBTERPlusGenerator.java b/src/main/java/ldbc/snb/bteronhplus/hadoop/HadoopBTERPlusGenerator.java
new file mode 100644
index 0000000..4ccd725
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/hadoop/HadoopBTERPlusGenerator.java
@@ -0,0 +1,116 @@
+package ldbc.snb.bteronhplus.hadoop;
+
+import ldbc.snb.bteronhplus.structures.Edge;
+import ldbc.snb.bteronhplus.tools.FileTools;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+/**
+ * Created by aprat on 16/08/16.
+ */
+public class HadoopBTERPlusGenerator {
+
+
+ public static class HadoopBTERPlusGeneratorMapper extends Mapper {
+
+
+ @Override
+ public void map(LongWritable key, Text value, Context context)
+ throws IOException, InterruptedException {
+
+ /*int threadId = Integer.parseInt(value.toString());
+
+ Configuration conf = context.getConfiguration();
+ String bterPlusStatsFile = conf.get("ldbc.snb.bteronh.generator.bterplusstats");
+ String offsetsFile = conf.get("ldbc.snb.bteronh.generator.offsets");
+ int numThreads = conf.getInt("ldbc.snb.bteronh.generator.numThreads",1);
+ int seed = conf.getInt("ldbc.snb.bteronh.generator.seed",0);
+
+ BTERPlusStats bterPlusStats[] = FileTools.deserializeObjectArray(bterPlusStatsFile,conf);
+ System.out.println("Number of plus stats "+bterPlusStats.length);
+ Long offsets[] = FileTools.deserializeObjectArray(offsetsFile,conf);
+
+ long totalExternalWeight = 0;
+ for(int i = 0; i < bterPlusStats.length; ++i) {
+ totalExternalWeight += bterPlusStats[i].totalExternalWeight;
+ }
+
+ double cumExternalDegreeProb[] = new double[bterPlusStats.length];
+ cumExternalDegreeProb[0] = bterPlusStats[0].totalExternalWeight / (double)totalExternalWeight;
+ for(int i = 1; i < bterPlusStats.length; ++i) {
+ cumExternalDegreeProb[i] = cumExternalDegreeProb[i-1] + bterPlusStats[i].totalExternalWeight / (double)totalExternalWeight;
+ }
+
+ long edgesToGenerate = (totalExternalWeight / 2);
+ long blockSize = edgesToGenerate / numThreads; // Dividing by two because externalWeightPerDegree is
+ // degree
+ if(threadId == 0) {
+ long residual = edgesToGenerate % numThreads;
+ blockSize += residual;
+ }
+
+ System.out.println("Mapper "+threadId+": Generating "+blockSize+" edges out of "+totalExternalWeight);
+ Random random = new Random();
+ random.setSeed(seed+threadId);
+ for(long i = 0; i < blockSize; ++i) {
+
+ int firstCommunity = BTER.sampleCumulative(cumExternalDegreeProb,random);
+ int secondCommunityIndex = BTER.sampleCumulative (bterPlusStats[firstCommunity]
+ .externalWeightPerBlockCumProb,random);
+ int secondCommunity = bterPlusStats[firstCommunity].blockIdMap[secondCommunityIndex];
+
+ int firstNodeDegree = BTER.sampleCumulative(bterPlusStats[firstCommunity]
+ .externalWeightPerDegreeCumProb, random) ;
+
+ long firstNode = bterPlusStats[firstCommunity].cumNumNodesPerDegree[firstNodeDegree] +
+ offsets[firstCommunity] + (long)(random.nextDouble()*bterPlusStats[firstCommunity]
+ .numNodesPerDegree[firstNodeDegree]);
+
+ int secondNodeDegree = BTER.sampleCumulative(bterPlusStats[secondCommunity]
+ .externalWeightPerDegreeCumProb, random) ;
+
+ long secondNode = bterPlusStats[secondCommunity].cumNumNodesPerDegree[secondNodeDegree] +
+ offsets[secondCommunity] + (long)(random.nextDouble()*bterPlusStats[secondCommunity]
+ .numNodesPerDegree[secondNodeDegree]);
+
+ Edge edge = new Edge(firstNode,secondNode);
+
+
+ context.write(new LongWritable(edge.getTail()), new LongWritable(edge.getHead()));
+ if( i % 100000 == 0 ) {
+ context.setStatus("Generated "+i+" edges out of "+blockSize+" in mapper "+threadId);
+ }
+ }
+ */
+ }
+ }
+
+
+ public static class HadoopBTERPlusGeneratorReducer extends Reducer {
+
+ @Override
+ public void reduce(LongWritable tail, Iterable valueSet,
+ Context context) throws IOException, InterruptedException {
+ /*Set neighbors = new HashSet();
+ for ( LongWritable head : valueSet ) {
+ neighbors.add(head.get());
+ }
+
+ for( Long head : neighbors) {
+ //String str = new String(tail + " " + head + "\n");
+ context.write(tail,new LongWritable(head));
+ }
+ */
+ }
+ }
+
+}
diff --git a/src/main/java/ldbc/snb/bteronh/hadoop/HadoopEdgePartitioner.java b/src/main/java/ldbc/snb/bteronhplus/hadoop/HadoopEdgePartitioner.java
similarity index 90%
rename from src/main/java/ldbc/snb/bteronh/hadoop/HadoopEdgePartitioner.java
rename to src/main/java/ldbc/snb/bteronhplus/hadoop/HadoopEdgePartitioner.java
index f603818..baefb68 100644
--- a/src/main/java/ldbc/snb/bteronh/hadoop/HadoopEdgePartitioner.java
+++ b/src/main/java/ldbc/snb/bteronhplus/hadoop/HadoopEdgePartitioner.java
@@ -1,4 +1,4 @@
-package ldbc.snb.bteronh.hadoop;
+package ldbc.snb.bteronhplus.hadoop;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Partitioner;
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/BasicCommunityStreamer.java b/src/main/java/ldbc/snb/bteronhplus/structures/BasicCommunityStreamer.java
new file mode 100644
index 0000000..1a346bb
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/BasicCommunityStreamer.java
@@ -0,0 +1,292 @@
+package ldbc.snb.bteronhplus.structures;
+
+import org.jgrapht.Graph;
+import org.jgrapht.alg.ConnectivityInspector;
+import org.jgrapht.alg.interfaces.SpanningTreeAlgorithm;
+import org.jgrapht.alg.spanning.BoruvkaMinimumSpanningTree;
+import org.jgrapht.graph.AsSubgraph;
+import org.jgrapht.graph.DefaultEdge;
+import org.jgrapht.graph.SimpleGraph;
+import org.jgrapht.graph.builder.GraphBuilder;
+
+import java.util.*;
+/*
+public class BasicCommunityStreamer implements CommunityStreamer {
+
+ protected static int NUM_MODELS_PER_SIZE = 10;
+
+ private Map> communityModels = null;
+ private List communities = null;
+ private GraphStats graphStats = null;
+ private Random random = null;
+ private int nextCommunityId = 0;
+
+ private static class Stub {
+ public int id;
+ public int degreeLeft;
+ public int degreeUsed;
+ public int totalDegree;
+ }
+
+ private static class CommunityModel {
+
+ private List degrees;
+ private double expectedDensity;
+ private double finalDensity;
+ private int excessDegree[];
+
+
+ public CommunityModel(List degrees, double expectedDensity) {
+ Collections.sort(degrees, new Comparator() {
+ @Override
+ public int compare(Double a, Double b) {
+ return ((int)(double)b) - ((int)(double)a);
+ }
+ });
+ this.degrees = degrees;
+ this.expectedDensity = expectedDensity;
+ this.finalDensity = 0.0;
+ }
+
+ public Community generate(int id) {
+
+ Random random = new Random();
+
+ Stub stubs[] = new Stub[degrees.size()];
+ for (int i = 0; i < degrees.size(); ++i) {
+ stubs[i] = new Stub();
+ stubs[i].id = i;
+ stubs[i].degreeLeft = (int)(double)degrees.get(i);
+ stubs[i].degreeUsed = 0;
+ stubs[i].totalDegree = (int)(double)degrees.get(i);
+ }
+
+ Arrays.sort(stubs, new Comparator() {
+ @Override
+ public int compare(Stub pair1, Stub pair2) {
+ return pair2.degreeLeft - pair1.degreeLeft;
+ }
+ });
+
+ List edges = new ArrayList();
+
+ for(int i = 0; i < stubs.length; ++i) {
+ for(int j = i+1; j < stubs.length; ++j) {
+ if(stubs[i].degreeLeft > 0 && stubs[j].degreeLeft > 0) {
+ Edge edge = new Edge(stubs[i].id, stubs[j].id);
+ stubs[i].degreeLeft--;
+ stubs[i].degreeUsed++;
+ stubs[j].degreeLeft--;
+ stubs[j].degreeUsed++;
+ edges.add(edge);
+ }
+ }
+ }
+
+ int numNodes = (int)(double)degrees.size();
+ int requiredEdges = (int)(numNodes*numNodes*expectedDensity)/2;
+ GraphBuilder builder = SimpleGraph.createBuilder(DefaultEdge.class);
+ for(Edge edge : edges) {
+ builder.addEdge(edge.getTail(), edge.getHead());
+ }
+ Graph graph = builder.build();
+ ConnectivityInspector connectivityInspector = new ConnectivityInspector<>(graph);
+ List> connectedComponents = connectivityInspector.connectedSets();
+ while(connectedComponents.size() > 1 ) {
+
+ connectedComponents.sort(new Comparator>() {
+ @Override
+ public int compare(Set component1, Set component2) {
+ return component2.size() - component1.size();
+ }
+ });
+
+ AsSubgraph first = new AsSubgraph<>(graph,connectedComponents.get(0));
+ AsSubgraph second = new AsSubgraph<>(graph,connectedComponents.get(1));
+
+ BoruvkaMinimumSpanningTree spanningTreeAlgorithm1 = new BoruvkaMinimumSpanningTree(first);
+ SpanningTreeAlgorithm.SpanningTree spanningTree1 = spanningTreeAlgorithm1.getSpanningTree();
+ BoruvkaMinimumSpanningTree spanningTreeAlgorithm2 = new BoruvkaMinimumSpanningTree(second);
+ SpanningTreeAlgorithm.SpanningTree spanningTree2 = spanningTreeAlgorithm2.getSpanningTree();
+
+ Set candidateEdges1 = new HashSet(first.edgeSet());
+ candidateEdges1.removeAll(spanningTree1.getEdges());
+
+ Set candidateEdges2 = new HashSet(second.edgeSet());
+ candidateEdges2.removeAll(spanningTree2.getEdges());
+
+ if(candidateEdges1.size() == 0 && candidateEdges2.size() == 0) {
+ return null;
+ }
+
+ ArrayList edgesArray1 = null;
+ if(candidateEdges1.size() > 0) {
+ edgesArray1 = new ArrayList(candidateEdges1);
+ } else {
+ edgesArray1 = new ArrayList(spanningTree1.getEdges());
+ }
+ ArrayList edgesArray2 = null;
+
+ if(candidateEdges2.size() > 0) {
+ edgesArray2 = new ArrayList(candidateEdges2);
+ } else {
+ edgesArray2 = new ArrayList<>(spanningTree2.getEdges());
+ }
+
+ Collections.shuffle(edgesArray1);
+ Collections.shuffle(edgesArray2);
+
+ while(edgesArray1.size() > 0 && edgesArray2.size() > 0) {
+
+ DefaultEdge edge1 = edgesArray1.get(edgesArray1.size()-1);
+ edgesArray1.remove(edgesArray1.size()-1);
+
+ DefaultEdge edge2 = edgesArray2.get(edgesArray2.size()-1);
+ edgesArray2.remove(edgesArray2.size()-1);
+
+
+ Long source1 = graph.getEdgeSource(edge1);
+ Long target1 = graph.getEdgeTarget(edge1);
+ Long source2 = graph.getEdgeSource(edge2);
+ Long target2 = graph.getEdgeTarget(edge2);
+
+ graph.removeEdge(edge1);
+ graph.removeEdge(edge2);
+ graph.addEdge(source1, source2);
+ graph.addEdge(target1, target2);
+
+ }
+
+ connectivityInspector = new ConnectivityInspector<>(graph);
+ connectedComponents = connectivityInspector.connectedSets();
+
+ }
+
+
+ BoruvkaMinimumSpanningTree spanningTreeAlgorithm = new BoruvkaMinimumSpanningTree(graph);
+ SpanningTreeAlgorithm.SpanningTree spanningTree = spanningTreeAlgorithm.getSpanningTree();
+ Set candidateEdges = new HashSet(graph.edgeSet());
+ candidateEdges.removeAll(spanningTree.getEdges());
+ ArrayList toShuffle = new ArrayList(candidateEdges);
+ Collections.shuffle(toShuffle);
+ while(!toShuffle.isEmpty()) {
+ DefaultEdge first = toShuffle.get(toShuffle.size()-1);
+ toShuffle.remove(toShuffle.size()-1);
+ if(!toShuffle.isEmpty()) {
+ DefaultEdge second = toShuffle.get(toShuffle.size()-1);
+ toShuffle.remove(toShuffle.size()-1);
+
+ Long source1 = graph.getEdgeSource(first);
+ Long target1 = graph.getEdgeTarget(first);
+ Long source2 = graph.getEdgeSource(second);
+ Long target2 = graph.getEdgeTarget(second);
+ Set set = new HashSet();
+ set.add(source1);
+ set.add(target1);
+ set.add(source2);
+ set.add(target2);
+ if (set.size() == 4) {
+ graph.removeEdge(first);
+ graph.removeEdge(second);
+ graph.addEdge(source1, source2);
+ graph.addEdge(target1, target2);
+ }
+ }
+ }
+
+ spanningTreeAlgorithm = new BoruvkaMinimumSpanningTree(graph);
+ spanningTree = spanningTreeAlgorithm.getSpanningTree();
+ candidateEdges = new HashSet(graph.edgeSet());
+ candidateEdges.removeAll(spanningTree.getEdges());
+ ArrayList toRemove = new ArrayList(candidateEdges);
+ Collections.shuffle(toRemove);
+ int edgesToRemove = Math.max(graph.edgeSet().size() - requiredEdges,1);
+ while(!toRemove.isEmpty() && edgesToRemove > 0) {
+ graph.removeEdge(toRemove.get(toRemove.size()-1));
+ toRemove.remove(toRemove.size()-1);
+ edgesToRemove--;
+ }
+
+ stubs = new Stub[degrees.size()];
+
+ for (int i = 0; i < degrees.size(); ++i) {
+ stubs[i] = new Stub();
+ stubs[i].id = i;
+ stubs[i].degreeLeft = (int)(double)degrees.get(i);
+ stubs[i].degreeUsed = 0;
+ stubs[i].totalDegree = (int)(double)degrees.get(i);
+ }
+ Set finalEdges = graph.edgeSet();
+
+ edges = new ArrayList();
+ for(DefaultEdge edge : finalEdges ) {
+ Long source = graph.getEdgeSource(edge);
+ Long target = graph.getEdgeTarget(edge);
+ stubs[(int)(long)source].degreeLeft--;
+ stubs[(int)(long)source].degreeUsed++;
+
+ stubs[(int)(long)target].degreeLeft--;
+ stubs[(int)(long)target].degreeUsed++;
+ edges.add(new Edge(source,target));
+ }
+
+
+ int excessDegree[] = new int[degrees.size()];
+ for(int i = 0; i < stubs.length; ++i ) {
+ excessDegree[i]=stubs[i].degreeLeft;
+ }
+
+ return new Community(id, degrees.size(), edges, excessDegree);
+ }
+
+ public List getDegrees() {
+ return degrees;
+ }
+ }
+
+ public BasicCommunityStreamer(GraphStats graphStats) {
+ communityModels = new HashMap>();
+ communities = new ArrayList();
+ this.graphStats = graphStats;
+ this.random = new Random();
+ this.nextCommunityId = 0;
+
+ for(Integer size : graphStats.getCommunitySizes()) {
+ System.out.println("Generating models for size: "+size);
+ ArrayList models = new ArrayList();
+ while(models.size() < NUM_MODELS_PER_SIZE) {
+ EmpiricalDistribution degreeDistribution = graphStats.getDegreeDistribution(size);
+ List degrees = new ArrayList();
+ for (int i = 0; i < size; ++i) {
+ double nextDegree = degreeDistribution.getNext();
+ degrees.add(nextDegree);
+ }
+
+ double density = (double) graphStats.getCommunityDensity(size);
+ CommunityModel model = new CommunityModel(degrees, density);
+
+ Community community = model.generate(nextCommunityId);
+ if(community != null) {
+ models.add(model);
+ }
+ }
+ communityModels.put(size, models);
+ }
+ }
+
+ @Override
+ public Community getModel(int id) {
+ return null;
+ }
+
+ @Override
+ public Community next() {
+ int nextCommunitySize = (int) graphStats.getCommunitySizeDistribution().getNext();
+ CommunityModel model = null;
+ model = communityModels.get(nextCommunitySize).get(random.nextInt(NUM_MODELS_PER_SIZE));
+ Community community = model.generate(nextCommunityId);
+ nextCommunityId+=1;
+ return community;
+ }
+}
+*/
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/BlockModel.java b/src/main/java/ldbc/snb/bteronhplus/structures/BlockModel.java
new file mode 100644
index 0000000..6b0fe17
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/BlockModel.java
@@ -0,0 +1,90 @@
+package ldbc.snb.bteronhplus.structures;
+
+import java.util.*;
+
+public class BlockModel {
+
+ final static String SEPARATOR = "\\|";
+ final static float CLUSTER_THRESHOLD = 0.8f;
+ final static int MAX_BLOCKS = 2000;
+
+ public static class ModelEntry {
+
+ public int id;
+ public double size;
+ public HashMap degree = new HashMap();
+ public double totalDegree = 0.0;
+ public double externalDegree = 0.0;
+
+
+ public ModelEntry() {
+ id = 0;
+ size = 0;
+ }
+
+ public ModelEntry(String entry) {
+ String fields[] = entry.split(SEPARATOR);
+ this.id = Integer.parseInt(fields[0]);
+ this.size = Double.parseDouble(fields[1]);
+ for( int i = 2; i < fields.length; ++i) {
+ String cell[] = fields[i].split(":");
+ int idOther = Integer.parseInt(cell[0]);
+ double degree = Double.parseDouble(cell[1]);
+ this.degree.put(idOther,degree);
+ totalDegree+=degree;
+ if(idOther != this.id) {
+ externalDegree += degree;
+ }
+ }
+ }
+
+ }
+
+ Map entries = new HashMap();
+ Map> children = new HashMap>();
+
+ public BlockModel(String blockmodel, String childrenmodel) {
+ String[] lines = blockmodel.split("\n");
+
+ // Parsing entries
+ for(int i = 0; i < lines.length; ++i) {
+ ModelEntry entry = new ModelEntry(lines[i]);
+ entries.put(entry.id, entry);
+ }
+
+ lines = childrenmodel.split("\n");
+ for(int i = 0; i < lines.length; ++i) {
+ String line[] = lines[i].split(SEPARATOR);
+ int id = Integer.parseInt(line[0]);
+ List blockChildren = children.get(id);
+ if(blockChildren == null) {
+ blockChildren = new ArrayList();
+ children.put(id,blockChildren);
+ }
+
+ for(int j = 1; j < line.length; ++j) {
+ blockChildren.add(Integer.parseInt(line[j]));
+ }
+ }
+ }
+
+ public static BlockModel identity() {
+ return new BlockModel("0|1.0|0:1.0", "0|0");
+ }
+
+ public Map getEntries() {
+ return entries;
+ }
+
+ public void setChildren(Map> children) {
+ this.children = children;
+ }
+
+ public int getNumBlocks() {
+ return entries.size();
+ }
+
+ public Map> getChildren() {
+ return children;
+ }
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/BlockModelCluster.java b/src/main/java/ldbc/snb/bteronhplus/structures/BlockModelCluster.java
new file mode 100644
index 0000000..176b198
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/BlockModelCluster.java
@@ -0,0 +1,126 @@
+package ldbc.snb.bteronhplus.structures;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
+
+/*
+public class BlockModelCluster implements SuperNode {
+
+ private List children = null;
+ private BlockModel childrenModel = null;
+ private List childrenBlocks = null;
+ private long id = 0;
+ private double sampleProbabilityExternal[] = null;
+ private long size = 0L;
+
+ private ArrayList entries;
+
+ private class Entry {
+ public Entry(long id1, long id2, double prob) {
+ this.id1 = id1;
+ this.id2 = id2;
+ this.prob = prob;
+ }
+ public long id1;
+ public long id2;
+ public double prob;
+ }
+
+
+
+ public BlockModelCluster(long id, List childrenBlocks, BlockModel childrenModel, List children) {
+ this.childrenModel = childrenModel;
+ this.childrenBlocks = childrenBlocks;
+ this.children = children;
+ this.id = id;
+ this.entries = new ArrayList();
+
+ Set availableChildren = new HashSet();
+ for(SuperNode child : children) {
+ availableChildren.add(child.getId());
+ this.size += child.getSize();
+ }
+
+ Set expectedChildren = new HashSet(childrenBlocks);
+
+ Map indices = new HashMap();
+ for(int i = 0; i < children.size(); ++i) {
+ indices.put(children.get(i).getId(), i);
+ }
+
+ double externalDegree[] = new double[children.size()];
+ Arrays.fill(externalDegree, 0.0);
+
+ double totalDegree = 0.0;
+ double totalInternalDegree = 0.0;
+ for(Long child : expectedChildren) {
+ BlockModel.ModelEntry entry = childrenModel.getEntries().get(child);
+ totalDegree += entry.totalDegree;
+ Integer index = indices.get(child);
+ for(Map.Entry neighbor : entry.degree.entrySet()) {
+ if(expectedChildren.contains(neighbor.getKey())) {
+ totalInternalDegree += neighbor.getValue();
+ if(entry.id <= neighbor.getKey()) {
+ entries.add(new Entry(entry.id, neighbor.getKey(), neighbor.getValue()));
+ }
+ } else {
+ if (index != null) {
+ externalDegree[index] += neighbor.getValue();
+ }
+ }
+ }
+ }
+
+ for(Entry entry : entries) {
+ entry.prob /= totalInternalDegree;
+ }
+
+ double totalExternalDegree = 0.0;
+ for(int i = 0; i < externalDegree.length; ++i) {
+ totalExternalDegree += externalDegree[i];
+ }
+
+
+ sampleProbabilityExternal = new double[children.size()];
+ Arrays.fill(sampleProbabilityExternal, 0.0);
+ sampleProbabilityExternal[0] = 0.0;
+ for(int i = 1; i < sampleProbabilityExternal.length; ++i) {
+ sampleProbabilityExternal[i] = sampleProbabilityExternal[i-1] + externalDegree[i] / totalExternalDegree;
+ }
+
+
+ }
+
+
+ @Override
+ public long getSize() {
+ return 0;
+ }
+
+ @Override
+ public long getInternalDegree() {
+ return 0;
+ }
+
+ @Override
+ public long getExternalDegree() {
+ return 0;
+ }
+
+ @Override
+ public int getId() {
+ return 0;
+ }
+
+ @Override
+ public void sampleEdges(FileWriter writer, Random random, long numEdges, long offset) throws IOException {
+
+ }
+
+ @Override
+ public long sampleNode(Random random, long offset) {
+ return 0;
+ }
+}
+ */
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/BlockSampler.java b/src/main/java/ldbc/snb/bteronhplus/structures/BlockSampler.java
new file mode 100644
index 0000000..29c68a3
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/BlockSampler.java
@@ -0,0 +1,282 @@
+package ldbc.snb.bteronhplus.structures;
+
+import org.apache.commons.math3.util.Pair;
+import org.jgrapht.graph.builder.GraphBuilder;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
+
+public class BlockSampler {
+
+ private int modelId [] = null;
+ private long degreePerModelType[] = null;
+ private double cumulativeProb[] = null;
+ private Map offsets = null;
+ private long numModels [] = null;
+ private CommunityStreamer streamer = null;
+ private long numCommunities = 0L;
+ private Map counts = null;
+
+
+
+ public BlockSampler(Map counts,
+ CommunityStreamer streamer) {
+
+ this.counts = counts;
+ this.streamer = streamer;
+ this.numModels = new long[counts.keySet().size()];
+ this.degreePerModelType = new long[counts.keySet().size()];
+ Arrays.fill(degreePerModelType, 0L);
+ this.modelId = new int[degreePerModelType.length];
+ this.offsets = new HashMap();
+
+ if(modelId.length > 0) {
+
+ long currentOffset = 0;
+ long totalExternalDegree = 0;
+ int index = 0;
+ for (Map.Entry entry : counts.entrySet()) {
+ Community model = streamer.getModel(entry.getKey());
+ modelId[index] = model.getId();
+ degreePerModelType[index] = model.getExternalDegree() * entry.getValue();
+ totalExternalDegree += degreePerModelType[index];
+ this.offsets.put(model.getId(), currentOffset);
+ this.numModels[index] = entry.getValue();
+ currentOffset += model.getSize() * entry.getValue();
+ numCommunities += entry.getValue();
+ index++;
+ }
+
+ cumulativeProb = new double[degreePerModelType.length];
+ cumulativeProb[0] = 0.0;
+ for (int i = 1; i < cumulativeProb.length; ++i) {
+ cumulativeProb[i] = cumulativeProb[i - 1] + degreePerModelType[i - 1] / (double) totalExternalDegree;
+ }
+
+ }
+
+ }
+
+ public long sample(Random random, long offset) {
+
+ if(modelId.length > 0) {
+
+ int pos = Arrays.binarySearch(cumulativeProb, random.nextDouble());
+
+ if (pos < 0) {
+ pos = -(pos + 1);
+ pos--;
+ }
+
+ /*if (pos == -1) {
+ pos = 0;
+ }*/
+
+ Community model = streamer.getModel(modelId[pos]);
+ long modelOffset = offsets.get(modelId[pos]);
+ long instanceIndex = random.nextLong() % numModels[pos];
+ modelOffset += model.getSize() * instanceIndex;
+ long finalOffset = offset + modelOffset;
+ return model.sampleNode(random, finalOffset);
+ }
+ return -1;
+
+ }
+
+ public void generateConnectedGraph(FileWriter writer, Random random, long baseOffset, GraphBuilder builder) throws
+ IOException {
+
+ if(numModels.length > 1) {
+ ArrayList> models = new ArrayList>();
+ for (int i = 0; i < numModels.length; ++i) {
+ Community model = streamer.getModel(modelId[i]);
+ long nextOffset = baseOffset + offsets.get(modelId[i]);
+ for (int j = 0; j < numModels[i]; ++j) {
+ if(model.getExternalDegree() > 0) {
+ models.add(new Pair(model, nextOffset));
+ nextOffset += models.size();
+ }
+ }
+ }
+
+ models.sort(new Comparator>() {
+
+ @Override
+ public int compare(Pair pair1, Pair pair2) {
+ long externalDegree1 = pair1.getFirst().getExternalDegree();
+ long externalDegree2 = pair2.getFirst().getExternalDegree();
+ if(externalDegree1 > externalDegree2) return -1;
+ if(externalDegree1 == externalDegree2) return 0;
+ return 1;
+ }
+ });
+
+
+
+
+ if(models.size() > 0) {
+ Community root = models.get(0).getFirst();
+ for (int i = 1; i < models.size(); ++i) {
+ Community nextModel = models.get(i).getFirst();
+ long node1 = root.sampleNode(random, models.get(0).getSecond());
+ long node2 = nextModel.sampleNode(random, models.get(i).getSecond());
+
+ writer.write(node1 + "\t" + node2 + "\n");
+ if(node1 != node2)
+ builder.addEdge(node1, node2);
+ }
+ }
+ }
+
+
+ }
+
+ public void generateCommunityEdges(FileWriter writer,
+ Map counts,
+ CommunityStreamer communityStreamer,
+ long offset,
+ GraphBuilder builder) throws IOException {
+
+ for(Map.Entry entry : counts.entrySet()) {
+ Community model = communityStreamer.getModel(entry.getKey());
+ long currentOffset = offset + offsets.get(entry.getKey());
+ for(int i = 0; i < entry.getValue(); ++i) {
+ for(Edge edge : model.getEdges()) {
+ long tail = (currentOffset + edge.getTail());
+ long head = (currentOffset + edge.getHead());
+ writer.write( tail + "\t" + head + "\n");
+ builder.addEdge(tail,head);
+ }
+ currentOffset += model.getSize();
+ }
+ }
+
+ }
+
+ private class NodeStats{
+ public long nodeId;
+ public int numTriangles;
+ public int degree;
+
+ public NodeStats(long nodeId, int numTriangles, int degree) {
+ this.nodeId = nodeId;
+ this.numTriangles = numTriangles;
+ this.degree = degree;
+ }
+ }
+
+ class Bucket {
+ private List nodes = new ArrayList();
+ int nmax = Integer.MAX_VALUE;
+
+ public boolean addNode(NodeStats node) {
+ nodes.add(node);
+ nmax = Math.min(node.degree + 1, nmax);
+ return nmax <= nodes.size();
+ }
+
+ public List getNodes() {
+ return nodes;
+ }
+
+ public int getNmax() {
+ return nmax;
+ }
+
+ public boolean merge(Bucket bucket) {
+ nmax = Math.min(nmax,bucket.getNmax());
+ nodes.addAll(bucket.getNodes());
+ return nmax <= nodes.size();
+ }
+ }
+
+
+ public long darwini(FileWriter writer,
+ Random random,
+ long offset) throws IOException {
+
+ List filledBuckets = new ArrayList();
+ HashMap currentBuckets = new HashMap();
+ List nodes = new ArrayList();
+ for(int i = 0; i < modelId.length; ++i ) {
+ Community model = streamer.getModel(modelId[i]);
+ long baseOffset = offset + offsets.get(modelId[i]);
+ for(int j = 0; j < numModels[i]; ++j) {
+ for(int k = 0; k < model.getExternalTriangles().size(); ++k) {
+ int numTriangles = model.getExternalTriangles().get(k);
+ if(numTriangles > 0) {
+ int degree = model.getExcessDegree().get(k);
+ nodes.add(new NodeStats(k + baseOffset, numTriangles, degree));
+ }
+ }
+ baseOffset+=model.getSize();
+ }
+ }
+
+ int index = 0;
+ for(NodeStats node : nodes) {
+ Bucket bucket = currentBuckets.get(node.numTriangles);
+ if(bucket == null) {
+ bucket = new Bucket();
+ currentBuckets.put(node.numTriangles, bucket);
+ }
+
+ if(bucket.addNode(node)) {
+ currentBuckets.remove(node.numTriangles);
+ filledBuckets.add(bucket);
+ }
+
+ index++;
+ }
+
+ List toMergeBuckets = new ArrayList(currentBuckets.values());
+ toMergeBuckets.sort(new Comparator() {
+ @Override
+ public int compare(Bucket bucket1, Bucket bucket2) {
+ return bucket1.getNodes().size() - bucket2.getNodes().size();
+ }
+ });
+
+ Bucket bucket = new Bucket();
+
+ for(Bucket nextBucket : toMergeBuckets) {
+ if(bucket.merge(nextBucket)) {
+ filledBuckets.add(bucket);
+ bucket = new Bucket();
+ }
+ }
+
+ if(bucket.getNodes().size() > 0) {
+ filledBuckets.add(bucket);
+ }
+
+ long numGeneratedEdges = 0L;
+ for(Bucket nextBucket : filledBuckets) {
+ long numTriangles = nextBucket.getNodes().get(0).numTriangles;
+ long size = nextBucket.getNodes().size();
+ if(size > 2) {
+ double prob = Math.pow(2.0 * numTriangles / (double) ((size - 1) * (size - 2)), 1 / 3.0);
+ for (int i = 0; i < size; i++) {
+ for (int j = i + 1; j < size; j++) {
+ if (random.nextDouble() < prob) {
+ long tail = nextBucket.getNodes().get(i).nodeId;
+ long head = nextBucket.getNodes().get(j).nodeId;
+ writer.write(tail + "\t" + head + "\n");
+ numGeneratedEdges++;
+ }
+ }
+ }
+ }
+ }
+ return numGeneratedEdges;
+
+ }
+ public long getNumCommunities() {
+ return numCommunities;
+ }
+
+ public Map getCounts() {
+ return counts;
+ }
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/Community.java b/src/main/java/ldbc/snb/bteronhplus/structures/Community.java
new file mode 100644
index 0000000..6b2f4a8
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/Community.java
@@ -0,0 +1,153 @@
+package ldbc.snb.bteronhplus.structures;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
+
+import org.jgrapht.Graph;
+import org.jgrapht.alg.ConnectivityInspector;
+import org.jgrapht.graph.DefaultEdge;
+import org.jgrapht.graph.SimpleGraph;
+import org.jgrapht.graph.builder.GraphBuilder;
+
+public class Community implements SuperNode {
+ private int id;
+ private List edges;
+ private List excessDegree;
+ private List clusteringCoefficient;
+ private double excessDegreeProbs[];
+ private int excessDegreeIndices[];
+ private int internalDegree = 0;
+ private int externalDegree = 0;
+ private List externalTriangles;
+
+ public Community(int id,
+ List excessDegree,
+ List clusteringCoefficient,
+ List edges) {
+ this.id = id;
+ this.edges = edges;
+ this.excessDegree = excessDegree;
+ this.clusteringCoefficient = clusteringCoefficient;
+ this.externalTriangles = new ArrayList();
+ this.internalDegree = edges.size()*2;
+ this.excessDegreeIndices = new int[excessDegree.size()];
+ int excessDegreeWitohutTriangles[] = new int[excessDegree.size()];
+ long degreeWithoutTriangles = 0L;
+
+ GraphBuilder builder = SimpleGraph.createBuilder(DefaultEdge.class);
+ for(int i = 0; i < excessDegree.size(); ++i) {
+ builder.addVertex(new Long(i));
+ }
+
+ for(Edge edge : edges) {
+ builder.addEdge(edge.getTail(), edge.getHead());
+ }
+
+ Graph graph = builder.build();
+ for(int i = 0; i < excessDegree.size(); ++i) {
+ int numTriangles = 0;
+ Set neighborEdges = graph.outgoingEdgesOf(new Long(i));
+ Set neighbors = new HashSet();
+ for(DefaultEdge edge : neighborEdges) {
+ neighbors.add(graph.getEdgeSource(edge));
+ neighbors.add(graph.getEdgeTarget(edge));
+ }
+ neighbors.remove(new Long(i));
+
+ for(Long neighbor : neighbors) {
+
+ for(DefaultEdge edge2 : graph.outgoingEdgesOf(neighbor)) {
+ if(neighbors.contains(graph.getEdgeSource(edge2)) &&
+ neighbors.contains(graph.getEdgeTarget(edge2))) {
+ numTriangles++;
+ }
+ }
+ }
+ int degree = graph.degreeOf(new Long(i)) + excessDegree.get(i);
+ int missingTriangles = (int)(clusteringCoefficient.get(i)*degree*(degree-1) - numTriangles) / 2;
+ externalTriangles.add(missingTriangles);
+ }
+
+ int next = 0;
+ for(int i = 0; i < excessDegree.size(); ++i) {
+ int nextDegree = excessDegree.get(i);
+ this.externalDegree += nextDegree;
+ nextDegree -= Math.sqrt(2.0*externalTriangles.get(i));
+ excessDegreeWitohutTriangles[i] = nextDegree;
+ degreeWithoutTriangles += nextDegree;
+ if(nextDegree > 0) {
+ excessDegreeIndices[next] = i;
+ next++;
+ }
+ }
+
+ this.excessDegreeProbs = new double[next];
+ if(next > 0) {
+ excessDegreeProbs[0] = 0.0;
+ for (int i = 1; i < next; ++i) {
+ excessDegreeProbs[i] = excessDegreeProbs[i - 1] + excessDegreeWitohutTriangles[excessDegreeIndices[i - 1]] /
+ (double) degreeWithoutTriangles;
+ }
+ }
+
+ }
+
+ public List getEdges() {
+ return edges;
+ }
+
+ @Override
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public long getSize() {
+ return excessDegree.size();
+ }
+
+ @Override
+ public long getInternalDegree() {
+ return internalDegree;
+ }
+
+ @Override
+ public long getExternalDegree() {
+ return externalDegree;
+ }
+
+ public List getExternalTriangles() {
+ return externalTriangles;
+ }
+
+ public List getExcessDegree() {
+ return excessDegree;
+ }
+
+ @Override
+ public void sampleEdges(FileWriter writer, Random random, long numEdges, long offset) throws IOException {
+
+ for(Edge edge : edges) {
+ writer.write((offset + edge.getTail()) + "\t" + (offset + edge.getHead()) + "\n");
+ }
+ }
+
+ @Override
+ public long sampleNode(Random random, long offset) {
+
+ int pos = Arrays.binarySearch(excessDegreeProbs, random.nextDouble());
+
+ if(pos < 0) {
+ pos = -(pos+1);
+ pos--;
+ }
+
+ if(pos == -1) {
+ pos = 0;
+ }
+
+ return offset + excessDegreeIndices[pos];
+ }
+
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/CommunityCluster.java b/src/main/java/ldbc/snb/bteronhplus/structures/CommunityCluster.java
new file mode 100644
index 0000000..acdb9e2
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/CommunityCluster.java
@@ -0,0 +1,306 @@
+package ldbc.snb.bteronhplus.structures;
+
+import org.apache.commons.math3.util.Pair;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
+
+public class CommunityCluster implements SuperNode {
+
+ private int id;
+ private List children;
+ private long offsets[] = null;
+ private int size = 0;
+ private long internalDegree = 0;
+ private long externalDegree = 0;
+ private double sampleNodeExternalCumProbability[];
+ private double sampleNodeInternalProbability[];
+ private double sampleInterNodeProbability[];
+ private long interDegree[];
+ private int sampleNodeExternalIndices[];
+ private double intraNodeEdgeProb;
+
+ private double expectedRatio = 0.0;
+
+ public CommunityCluster(int id,
+ List children,
+ double externalRatio
+ ) {
+ this.id = id;
+ this.expectedRatio = externalRatio;
+ if(id == 0) {
+ System.out.println();
+ }
+ this.children = new ArrayList(children);
+ this.children.sort( new Comparator() {
+ @Override
+ public int compare(SuperNode superNode1, SuperNode superNode2) {
+ return -1*(int)(superNode1.getExternalDegree()-superNode2.getExternalDegree());
+ }
+ });
+
+ this.offsets = new long[this.children.size()];
+
+ this.sampleNodeExternalIndices = new int[this.children.size()];
+
+ offsets[0] = 0;
+ for(int i = 1; i < offsets.length; ++i) {
+ offsets[i] = offsets[i-1] + this.children.get(i-1).getSize();
+ }
+
+ long tempInternalZeroExternalDegree = 0;
+ long tempInternalDegree = 0;
+ long tempExternalDegree = 0;
+ for(int i = 0; i < this.children.size(); ++i) {
+ SuperNode node = this.children.get(i);
+ tempInternalDegree += node.getInternalDegree();
+ tempExternalDegree += node.getExternalDegree();
+ if(node.getExternalDegree() == 0) {
+ tempInternalZeroExternalDegree += node.getInternalDegree();
+ }
+ size+=node.getSize();
+ }
+
+ if(id == 19) {
+ System.out.println();
+ }
+ long totalDegree = tempInternalDegree+tempExternalDegree;
+ double finalRatio = Math.min((totalDegree - tempInternalDegree )/ (double)(totalDegree), externalRatio);
+ boolean special = false;
+ if(externalRatio == 0.0 && tempExternalDegree > 0) {
+ finalRatio = (totalDegree - tempInternalDegree )/ (double)(totalDegree);
+ special = true;
+ }
+ long externalEdges[] = new long[this.children.size()];
+ Arrays.fill(externalEdges, 0);
+
+ long externalDegreeBudget = (long)(totalDegree*finalRatio);
+ /*double zeroExternalRatio = tempInternalZeroExternalDegree / (double)tempInternalDegree;
+ externalDegreeBudget -= externalDegreeBudget*zeroExternalRatio;*/
+ long totalToExternal = 0;
+
+ for(int i = 0; i < this.children.size() && externalDegreeBudget > 0; i++) {
+ SuperNode child = this.children.get(i);
+ long toRemove = 0;
+ if(!special) {
+ toRemove = Math.min((long)(child.getExternalDegree()*0.5),externalDegreeBudget);
+ //toRemove = Math.min(toRemove, externalDegreeBudget / 3);
+
+ } else {
+ toRemove = Math.min((long)(child.getExternalDegree()),externalDegreeBudget);
+ }
+ externalEdges[i] = toRemove;
+ externalDegreeBudget-=toRemove;
+ totalToExternal+=toRemove;
+ }
+ externalDegree=totalToExternal;
+
+ internalDegree = totalDegree-externalDegree;
+ long intraDegree = tempInternalDegree;
+ intraNodeEdgeProb = intraDegree / (double)(internalDegree);
+
+ int numExternalNonZero = 0;
+ for(int i = 0; i < this.children.size(); ++i) {
+ if(externalEdges[i] > 0) {
+ sampleNodeExternalIndices[numExternalNonZero] = i;
+ numExternalNonZero++;
+ }
+ }
+
+ sampleNodeExternalCumProbability = new double[numExternalNonZero];
+ sampleNodeInternalProbability = new double[this.children.size()];
+ sampleInterNodeProbability = new double[this.children.size()];
+ interDegree = new long[this.children.size()];
+
+ if(numExternalNonZero > 0) {
+ sampleNodeExternalCumProbability[0] = 0.0;
+ for (int i = 1; i < numExternalNonZero; ++i) {
+ sampleNodeExternalCumProbability[i] = sampleNodeExternalCumProbability[i - 1] +
+ externalEdges[sampleNodeExternalIndices[i - 1]] /
+ (double) externalDegree;
+ }
+ }
+
+ for (int i = 0; i < sampleInterNodeProbability.length; ++i) {
+ SuperNode child = this.children.get(i);
+ long childInternalDegree = child.getExternalDegree() - externalEdges[i];
+ interDegree[i] = childInternalDegree;
+ sampleInterNodeProbability[i] = (internalDegree - intraDegree) > 0 ? childInternalDegree /
+ (double) (internalDegree - intraDegree) : 0.0;
+ }
+
+ for (int i = 0; i < sampleNodeInternalProbability.length; ++i) {
+ SuperNode child = this.children.get(i);
+ long childInternalDegree = child.getInternalDegree();
+ sampleNodeInternalProbability[i] = childInternalDegree / (double) tempInternalDegree;
+ }
+
+
+ long childrenDegree = tempInternalDegree+tempExternalDegree;
+ long currentDegree = internalDegree + externalDegree;
+
+ if(childrenDegree != currentDegree) {
+ throw new RuntimeException("Found inconcistency between children and current degree "+currentDegree+" " +
+ ""+childrenDegree);
+ }
+
+ }
+
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public long getInternalDegree() {
+ return internalDegree;
+ }
+
+ @Override
+ public long getExternalDegree() {
+ return externalDegree;
+ }
+
+ @Override
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public void sampleEdges(FileWriter writer, Random random, long numEdges, long offset) throws IOException {
+
+ // Creating intra node edges
+ long intraNodeEdges = Math.round(intraNodeEdgeProb * numEdges);
+ for(int i = 0; i < this.children.size(); ++i) {
+ SuperNode child = this.children.get(i);
+ long numEdgesToCreate = (Math.round(intraNodeEdges * sampleNodeInternalProbability[i]));
+ child.sampleEdges(writer, random, numEdgesToCreate, offset + offsets[i]);
+ }
+
+
+ ArrayList> interDegrees = new ArrayList>();
+ for(int i = 0; i < interDegree.length; ++i) {
+ interDegrees.add(new Pair(i, interDegree[i]));
+ }
+
+ interDegrees.sort(new Comparator>() {
+
+ @Override
+ public int compare(Pair pair1, Pair pair2) {
+ if( pair1.getValue() > pair2.getValue()) return -1;
+ if( pair1.getValue() == pair2.getValue()) return 0;
+ return 1;
+ }
+
+ });
+
+
+ long degreeBudget[] = new long[interDegrees.size()];
+ int indices[] = new int[interDegrees.size()];
+
+ for(int i = 0; i < interDegrees.size(); ++i) {
+ degreeBudget[i] = interDegrees.get(i).getValue();
+ indices[i] = interDegrees.get(i).getKey();
+ }
+ if(id == 19) {
+ System.out.println();
+ }
+
+ boolean finished = false;
+ while(!finished) {
+ finished = true;
+ for (int i = 0; i < degreeBudget.length; ++i) {
+ for (int j = i + 1; j < degreeBudget.length && degreeBudget[i] > 0; ++j) {
+ if (degreeBudget[j] > 0) {
+ int index1 = indices[i];
+ int index2 = indices[j];
+ long node1 = children.get(index1).sampleNode(random, offset + offsets[index1]);
+ long node2 = children.get(index2).sampleNode(random, offset + offsets[index2]);
+ writer.write(node1 + "\t" + node2 + "\n");
+ degreeBudget[i]--;
+ degreeBudget[j]--;
+ finished = false;
+ }
+ }
+ }
+ }
+
+
+ int indicesNonZero[] = new int[indices.length];
+ int numNonZeros = 0;
+ long remainingDegree = 0;
+ for(int i = 0; i < degreeBudget.length; ++i) {
+ remainingDegree+=degreeBudget[i];
+ if(degreeBudget[i] != 0L) {
+ indicesNonZero[numNonZeros] = indices[i];
+ numNonZeros++;
+ }
+ }
+
+ if(numNonZeros > 0) {
+ double cumulative[] = new double[numNonZeros];
+ cumulative[0] = 0.0;
+ for (int i = 1; i < numNonZeros; ++i) {
+ double currentProb = degreeBudget[indicesNonZero[i - 1]] / (double) remainingDegree;
+ cumulative[i] = currentProb + cumulative[i - 1];
+ }
+
+
+ long numEdgesRemaining = remainingDegree / 2;
+ for (int i = 0; i < numEdgesRemaining; ++i) {
+
+ int pos = Arrays.binarySearch(cumulative, random.nextDouble());
+
+ if (pos < 0) {
+ pos = -(pos + 1);
+ pos--;
+ }
+
+ int first = indicesNonZero[pos];
+
+ pos = Arrays.binarySearch(cumulative, random.nextDouble());
+
+ if (pos < 0) {
+ pos = -(pos + 1);
+ pos--;
+ }
+
+ int second = indicesNonZero[pos];
+
+ long node1 = children.get(first).sampleNode(random, offset + offsets[first]);
+ long node2 = children.get(second).sampleNode(random, offset + offsets[second]);
+ writer.write(node1 + "\t" + node2 + "\n");
+ }
+ }
+
+ }
+
+ public List getChildren() {
+ return children;
+ }
+
+ @Override
+ public long sampleNode(Random random, long offset) {
+
+ int pos = Arrays.binarySearch(sampleNodeExternalCumProbability,random.nextDouble());
+
+ if(pos < 0) {
+ pos = -(pos+1);
+ pos--;
+ }
+
+ if(pos >= children.size()) {
+ pos = children.size() - 1;
+ }
+
+ if(pos == -1) {
+ pos = 0;
+ }
+
+ int index = sampleNodeExternalIndices[pos];
+ return children.get(index).sampleNode(random, offset + offsets[index]);
+ }
+
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/CommunityClusterStreamer.java b/src/main/java/ldbc/snb/bteronhplus/structures/CommunityClusterStreamer.java
new file mode 100644
index 0000000..4119270
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/CommunityClusterStreamer.java
@@ -0,0 +1,26 @@
+package ldbc.snb.bteronhplus.structures;
+
+import java.util.Iterator;
+import java.util.List;
+
+public class CommunityClusterStreamer implements CommunityStreamer {
+
+ private List clusters;
+ private Iterator iter;
+
+ public CommunityClusterStreamer(List clusters) {
+ this.clusters = clusters;
+ this.iter = clusters.iterator();
+ }
+
+ @Override
+ public Community getModel(int id) {
+ return null;
+ }
+
+ @Override
+ public Community next() {
+ return null;
+ //return iter.next();
+ }
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/CommunityStreamer.java b/src/main/java/ldbc/snb/bteronhplus/structures/CommunityStreamer.java
new file mode 100644
index 0000000..7a6c314
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/CommunityStreamer.java
@@ -0,0 +1,9 @@
+package ldbc.snb.bteronhplus.structures;
+
+public interface CommunityStreamer {
+
+ public abstract Community getModel(int id);
+
+ public abstract Community next();
+
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/CorePeripheryCommunityStreamer.java b/src/main/java/ldbc/snb/bteronhplus/structures/CorePeripheryCommunityStreamer.java
new file mode 100644
index 0000000..23aeb01
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/CorePeripheryCommunityStreamer.java
@@ -0,0 +1,408 @@
+package ldbc.snb.bteronhplus.structures;
+
+
+import java.util.*;
+
+/*
+
+public class CorePeripheryCommunityStreamer implements CommunityStreamer {
+
+ protected static int NUM_MODELS_PER_SIZE = 100;
+
+ private Map> communityModels = null;
+ private List communities = null;
+ private GraphStats graphStats = null;
+ private Random random = null;
+ private int nextCommunityId = 0;
+
+ private static class NodeInfo {
+ public int degree;
+ public int numTriangles;
+ public double clusteringCoefficient;
+ }
+
+
+ public static class CommunityModel {
+
+ public int size = 0;
+ public List nodeInfo = new ArrayList();
+ public int minDegree = Integer.MAX_VALUE;
+ public int minTriangles = Integer.MAX_VALUE;
+ public double coreDensity = 1.0;
+ public int coreSize = 0;
+ public double peripheryDensity[] = null;
+ public int peripheryCoreThreshold[] = null;
+ public double triangleBudget[] = null;
+ public double degreeBudget[] = null;
+ public int currentCoreThreshold = 0;
+ public int peripherySize = 0;
+
+ public CommunityModel(int size) {
+ this.size = size;
+ this.peripheryDensity = new double[size];
+ this.triangleBudget = new double[size];
+ this.degreeBudget = new double[size];
+ this.peripheryCoreThreshold = new int[size];
+ }
+
+ public Community generate(int id) {
+
+ Random random = new Random(id);
+ List edges = new ArrayList();
+ int numNodes = nodeInfo.size();
+ for(int i = 0; i < coreSize; ++i) {
+ for(int j = i+1; j < coreSize; ++j) {
+ double prob = random.nextDouble();
+ if(prob < coreDensity) {
+ edges.add(new Edge(i,j));
+ }
+ }
+ }
+
+ for(int i = 0; i < peripherySize; ++i) {
+ int peripheryIndex = coreSize+i;
+ for(int j = 0; j < peripheryCoreThreshold[peripheryIndex]; ++j) {
+ double prob = random.nextDouble();
+ if(prob < peripheryDensity[peripheryIndex]) {
+ edges.add(new Edge(peripheryIndex,j));
+ }
+ }
+ }
+
+ Map degree = new HashMap();
+ for(Edge edge : edges) {
+ degree.merge((int)edge.getTail(), 1 , Integer::sum);
+ degree.merge((int)edge.getHead(), 1 , Integer::sum);
+ }
+
+ int excessDegree[] = new int[numNodes];
+ for(Map.Entry entry : degree.entrySet()) {
+ excessDegree[entry.getKey()] = (int) Math.max(0.0, nodeInfo.get(entry.getKey()).degree - entry
+ .getValue());
+ }
+
+ return new Community(id, nodeInfo.size(), edges, excessDegree);
+ }
+
+ }
+
+ private static int findBucket(List buckets, NodeInfo nodeInfo, int start ) {
+
+ double bestScore = 0;
+ int bestBucket = -1;
+ for(int i = 0; i < buckets.size(); ++i) {
+ int index = (start + i) % buckets.size();
+ CommunityModel nextBucket = buckets.get(index);
+ if(nextBucket.nodeInfo.size() < nextBucket.size) {
+ int coreSize = nextBucket.coreSize;
+ int expectedTrianglesPerNode = (int)((nextBucket.coreSize-1)*(nextBucket.coreSize-2)*Math.pow(nextBucket
+ .coreDensity,3));
+ int expectedTrianglesPerNodeAfterInsert = (int)((nextBucket.coreSize)*(nextBucket.coreSize-1)*Math.pow
+ (nextBucket.coreDensity,3)) / 2;
+
+ if(coreSize < nodeInfo.degree &&
+ coreSize+1 < nextBucket.minDegree &&
+ expectedTrianglesPerNode <= nodeInfo.numTriangles &&
+ expectedTrianglesPerNodeAfterInsert <= nextBucket.minTriangles) {
+ if(bestBucket == -1) {
+ bestBucket = index;
+ return bestBucket;
+ }
+ }
+ }
+ }
+
+ return bestBucket;
+ }
+
+ private static void addToBucket(CommunityModel bucket, NodeInfo nodeInfo ) {
+
+ bucket.nodeInfo.add(nodeInfo);
+ bucket.minDegree = Math.min(nodeInfo.degree, bucket.minDegree);
+ bucket.minTriangles = Math.min(nodeInfo.numTriangles, bucket.minTriangles);
+ bucket.coreSize = bucket.nodeInfo.size();
+ }
+
+ private static boolean accommodateToCore(List buckets, NodeInfo nodeInfo) {
+
+
+ CommunityModel bestBucket = null;
+ double bestScore = 0.0;
+ double bestAlpha = 0.0;
+ boolean found = false;
+ for(CommunityModel bucket : buckets) {
+ if(bucket.nodeInfo.size() < bucket.size) {
+ int coreSize = bucket.coreSize;
+ double density = bucket.coreDensity;
+ double minDegree = Math.min(bucket.minDegree, nodeInfo.degree);
+ double minTriangles = Math.min(bucket.minTriangles, nodeInfo.numTriangles);
+ double minAlpha1 = (coreSize*density - minDegree)/(double)coreSize;
+ double minAlpha2 = density - Math.pow(2*minTriangles/((double)coreSize*(coreSize-1)), 1/3.0);
+ double minAlpha = Math.max(Math.max(minAlpha1, minAlpha2), 0.0);
+ double maxAlpha = density*(1 - Math.pow((coreSize-2)/(double)(coreSize+1), 1/3.0));
+ if(maxAlpha > minAlpha) {
+ if(!found) {
+ bestBucket = bucket;
+ bestAlpha = minAlpha;
+ bestScore = (coreSize*(coreSize-1))*((coreSize+1)*Math.pow((density-minAlpha), 3.0) -
+ (coreSize-2)*Math.pow(density,3.0));
+ }
+ found = true;
+ double newScore = (coreSize*(coreSize-1))*((coreSize+1)*Math.pow((density-minAlpha), 3.0) -
+ (coreSize-2)*Math.pow(density,3.0));
+ if(bestScore < newScore) {
+ bestBucket = bucket;
+ bestAlpha = minAlpha;
+ bestScore = newScore;
+ }
+
+ }
+ }
+ }
+
+
+ if(found) {
+ bestBucket.nodeInfo.add(nodeInfo);
+ bestBucket.coreSize++;
+ bestBucket.minDegree = Math.min(nodeInfo.degree, bestBucket.minDegree);
+ bestBucket.minTriangles = Math.min(nodeInfo.numTriangles, bestBucket.minTriangles);
+ bestBucket.coreDensity -= bestAlpha;
+ return true;
+ } else {
+
+ for(CommunityModel bucket : buckets) {
+ if(bucket.coreSize <=2 && bucket.nodeInfo.size() < bucket.size) {
+ bucket.nodeInfo.add(nodeInfo);
+ bucket.coreSize++;
+ bucket.minDegree = Math.min(nodeInfo.degree, bucket.minDegree);
+ bucket.minTriangles = Math.min(nodeInfo.numTriangles, bucket.minTriangles);
+ bucket.coreDensity -= bestAlpha;
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean addToPeriphery(List buckets, NodeInfo nodeInfo) {
+
+ for(CommunityModel bucket : buckets) {
+ if(bucket.nodeInfo.size() < bucket.size) {
+ if(nodeInfo.degree >= 2) {
+ int subCoreSize = bucket.currentCoreThreshold;
+ double density = bucket.coreDensity;
+ int numTriangles = nodeInfo.numTriangles;
+ int degree = Math.min(subCoreSize, nodeInfo.degree);
+ double requiredDensity = Math.sqrt(2 * numTriangles / (double) (density * degree * (degree - 1)));
+ if (requiredDensity <= 1.0) {
+ double numTrianglesClosedPerNode = (degree * (degree - 1) * Math.pow(requiredDensity, 2.0) * density /
+ (2.0 * degree));
+ double degreeRealizedPerNode = requiredDensity;
+ boolean realizable = true;
+ for(int i = 0; i < degree; ++i) {
+ if (bucket.triangleBudget[i] - numTrianglesClosedPerNode < 0 ||
+ bucket.degreeBudget[i] - degreeRealizedPerNode < 0) {
+ realizable = false;
+ break;
+ }
+ }
+
+ if(realizable) {
+ int peripheryIndex = bucket.coreSize + bucket.peripherySize;
+ bucket.peripheryDensity[peripheryIndex] = requiredDensity;
+ bucket.peripheryCoreThreshold[peripheryIndex] = degree;
+ bucket.nodeInfo.add(nodeInfo);
+ bucket.peripherySize++;
+
+ for (int i = bucket.currentCoreThreshold - 1; i >= 0; --i) {
+ bucket.triangleBudget[i] -= numTrianglesClosedPerNode;
+ bucket.degreeBudget[i] -= degreeRealizedPerNode;
+ if (bucket.triangleBudget[i] <= 0 ||
+ bucket.degreeBudget[i] <= 0) {
+ bucket.currentCoreThreshold--;
+ }
+ }
+
+ return true;
+ }
+ }
+ } else {
+ double degreeRealizedPerNode = 1/(double)bucket.currentCoreThreshold;
+ int coreThresholdIndex = bucket.currentCoreThreshold - 1;
+ int numTrianglesClosedPerNode = 0;
+ if (coreThresholdIndex >= 0 && bucket.triangleBudget[coreThresholdIndex] -
+ numTrianglesClosedPerNode > 0 &&
+ bucket.degreeBudget[coreThresholdIndex] - degreeRealizedPerNode > 0) {
+
+ int peripheryIndex = bucket.coreSize + bucket.peripherySize;
+ bucket.peripheryDensity[peripheryIndex] = 1.0;
+ bucket.peripheryCoreThreshold[peripheryIndex] = bucket.currentCoreThreshold;
+ bucket.nodeInfo.add(nodeInfo);
+ bucket.peripherySize++;
+ for (int i = bucket.currentCoreThreshold - 1; i >= 0; --i) {
+ bucket.triangleBudget[i] -= numTrianglesClosedPerNode;
+ bucket.degreeBudget[i] -= degreeRealizedPerNode;
+ if (bucket.triangleBudget[i] <= 1 ||
+ bucket.degreeBudget[i] <= 1) {
+ bucket.currentCoreThreshold--;
+ }
+ }
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+ }
+
+
+
+ public static List createModels(int modelSize,
+ int numModels,
+ List degree,
+ List clusteringCoefficient) {
+
+ List buckets = new ArrayList();
+ for(int i = 0; i < numModels; ++i) {
+ buckets.add(new CommunityModel(modelSize));
+ }
+
+ ArrayList nodeInfos = new ArrayList();
+ for(int i = 0; i < degree.size(); ++i) {
+ NodeInfo nodeInfo = new NodeInfo();
+ nodeInfo.degree = degree.get(i);
+ nodeInfo.numTriangles = (int)(nodeInfo.degree*(nodeInfo.degree-1)*clusteringCoefficient.get(i)/2.0);
+ nodeInfo.clusteringCoefficient = clusteringCoefficient.get(i);
+ nodeInfos.add(nodeInfo);
+ }
+
+ nodeInfos.sort( new Comparator() {
+ @Override
+ public int compare(NodeInfo node1, NodeInfo node2) {
+ if(node1.numTriangles != node2.numTriangles) {
+ return node1.numTriangles - node2.numTriangles;
+ }
+
+ return node1.degree - node2.degree;
+
+ }
+ });
+
+ // Initialize buckets with 3 nodes each
+ int numToDistribute = 3*buckets.size();
+ if(buckets.get(0).size*buckets.size() < numToDistribute) {
+ numToDistribute = buckets.size();
+ }
+
+ for(int i = 0; i < numToDistribute; ++i) {
+ CommunityModel bucket = buckets.get(i%buckets.size());
+ //CommunityModel bucket = buckets.get(i/Math.min(buckets.get(0).size, 3));
+ NodeInfo nextNode = nodeInfos.remove(nodeInfos.size()-1);
+ bucket.nodeInfo.add(nextNode);
+ bucket.minDegree = Math.min(nextNode.degree, bucket.minDegree);
+ bucket.minTriangles = Math.min(nextNode.numTriangles, bucket.minTriangles);
+ bucket.coreSize++;
+ }
+
+
+ int start = 0;
+ List remainingNodes = new ArrayList();
+ while(!nodeInfos.isEmpty()) {
+ NodeInfo nextNode = nodeInfos.remove(nodeInfos.size()-1);
+ int index = findBucket(buckets, nextNode, start % buckets.size());
+ if(index >= 0) {
+ addToBucket(buckets.get(index), nextNode);
+ } else if(!accommodateToCore(buckets, nextNode)) {
+ remainingNodes.add(nextNode);
+ }
+ }
+
+
+ List peripheryNodes = new ArrayList(remainingNodes);
+
+ for(CommunityModel bucket : buckets) {
+ int coreSize = bucket.coreSize;
+ double density = bucket.coreDensity;
+ for(int i = 0; i < bucket.nodeInfo.size(); ++i) {
+ NodeInfo nextNode = bucket.nodeInfo.get(i);
+ bucket.triangleBudget[i] = nextNode.numTriangles - ((coreSize-1)*(coreSize-2)*Math.pow(density,3)/2.0);
+ bucket.degreeBudget[i] = nextNode.degree - (coreSize-1)*density;
+ }
+ bucket.currentCoreThreshold = bucket.coreSize;
+ for(int j = bucket.coreSize-1;
+ j >=0 && (bucket.triangleBudget[j] <= 0 || bucket.degreeBudget[j] <= 0);
+ --j, --bucket.currentCoreThreshold) {
+ }
+ }
+
+ remainingNodes.clear();
+ for(NodeInfo nextNode : peripheryNodes) {
+ if (!addToPeriphery(buckets, nextNode)) {
+ remainingNodes.add(nextNode);
+ }
+ }
+
+ for(int i = 0; i < buckets.size(); ++i) {
+ CommunityModel nextBucket = buckets.get(i);
+ if(nextBucket.nodeInfo.size() < nextBucket.size) {
+ int numMissing = nextBucket.size - nextBucket.nodeInfo.size();
+ for(int j = 0; j < numMissing; ++j) {
+ NodeInfo nextNode = remainingNodes.remove(remainingNodes.size()-1);
+ nextBucket.nodeInfo.add(nextNode);
+ nextBucket.degreeBudget[0]--;
+ nextBucket.peripherySize++;
+ nextBucket.peripheryDensity[nextBucket.nodeInfo.size()-1] = 1.0;
+ nextBucket.peripheryCoreThreshold[nextBucket.nodeInfo.size()-1] = 1;
+ }
+ }
+ }
+
+ return buckets;
+
+ }
+
+ public CorePeripheryCommunityStreamer(GraphStats graphStats, Random random) {
+ this.random = random;
+ communityModels = new HashMap>();
+ communities = new ArrayList();
+ this.graphStats = graphStats;
+ this.nextCommunityId = 0;
+
+ for(Integer size : graphStats.getCommunitySizes()) {
+ System.out.println("Generating models for size: "+size);
+
+ EmpiricalDistribution degreeDistribution = graphStats.getDegreeDistribution(size);
+ int sequenceSize = NUM_MODELS_PER_SIZE*size;
+ List degree = new ArrayList();
+ List clusteringCoefficient = new ArrayList();
+ for (int i = 0; i < sequenceSize; ++i) {
+ double nextDegree = degreeDistribution.getNext();
+ degree.add((int)nextDegree);
+ clusteringCoefficient.add(graphStats.getCCPerDegreeDistribution((int)nextDegree).getNext());
+ }
+ List models = createModels(size,
+ NUM_MODELS_PER_SIZE,
+ degree,
+ clusteringCoefficient);
+ communityModels.put(size, models);
+ }
+ }
+
+
+ @Override
+ public Community getModel(int id) {
+ return null;
+ }
+
+ @Override
+ public Community next() {
+ int nextCommunitySize = (int) graphStats.getCommunitySizeDistribution().getNext();
+ CommunityModel model = null;
+ model = communityModels.get(nextCommunitySize).get(random.nextInt(NUM_MODELS_PER_SIZE));
+ Community community = model.generate(nextCommunityId);
+ nextCommunityId+=1;
+ return community;
+ }
+}
+*/
diff --git a/src/main/java/ldbc/snb/bteronh/structures/Edge.java b/src/main/java/ldbc/snb/bteronhplus/structures/Edge.java
similarity index 96%
rename from src/main/java/ldbc/snb/bteronh/structures/Edge.java
rename to src/main/java/ldbc/snb/bteronhplus/structures/Edge.java
index 7884f3f..c7494fd 100644
--- a/src/main/java/ldbc/snb/bteronh/structures/Edge.java
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/Edge.java
@@ -1,4 +1,4 @@
-package ldbc.snb.bteronh.structures;
+package ldbc.snb.bteronhplus.structures;
import org.apache.hadoop.io.WritableComparable;
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/EmpiricalDistribution.java b/src/main/java/ldbc/snb/bteronhplus/structures/EmpiricalDistribution.java
new file mode 100644
index 0000000..734b84b
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/EmpiricalDistribution.java
@@ -0,0 +1,70 @@
+package ldbc.snb.bteronhplus.structures;
+
+import ldbc.snb.bteronhplus.tools.FileTools;
+import org.apache.hadoop.conf.Configuration;
+import umontreal.iro.lecuyer.probdist.EmpiricalDist;
+import umontreal.iro.lecuyer.randvar.RandomVariateGen;
+import umontreal.iro.lecuyer.rng.LFSR113;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+
+public class EmpiricalDistribution {
+
+ RandomVariateGen randomVariateGen = null;
+
+ public EmpiricalDistribution(String filename, Configuration conf, double min) {
+
+ // reading the sequence from the file
+ ArrayList sequence = new ArrayList();
+ BufferedReader reader = null;
+ try {
+ reader = FileTools.getFile(filename, conf);
+ String line;
+ line = reader.readLine();
+ while(line!=null) {
+ double next = Double.parseDouble(line);
+ if(next >= min) {
+ sequence.add(next);
+ }
+ line = reader.readLine();
+ }
+ } catch (IOException e){
+ e.printStackTrace();
+ System.exit(1);
+ }
+ initialize(sequence);
+ }
+
+ public EmpiricalDistribution(ArrayList sequence) {
+ initialize(sequence);
+ }
+
+ private void initialize(ArrayList sequence ) {
+ Collections.sort(sequence);
+
+ double [] finalSequence = new double[sequence.size()];
+ for(int i = 0; i < sequence.size(); ++i){
+ finalSequence[i] = sequence.get(i);
+ }
+ EmpiricalDist distribution = new EmpiricalDist(finalSequence);
+
+
+ LFSR113 random = new LFSR113();
+ /*int [] seeds = new int[4];
+ seeds[0] = 128+sequence.size();
+ seeds[1] = 128+sequence.size()+1;
+ seeds[2] = 128+sequence.size()+2;
+ seeds[3] = 128+sequence.size()+3;
+ random.setSeed(seeds);*/
+ randomVariateGen = new RandomVariateGen(random,distribution);
+ }
+
+ public double getNext() {
+ return randomVariateGen.nextDouble();
+ }
+
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/GraphStats.java b/src/main/java/ldbc/snb/bteronhplus/structures/GraphStats.java
new file mode 100644
index 0000000..17bb9a7
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/GraphStats.java
@@ -0,0 +1,144 @@
+package ldbc.snb.bteronhplus.structures;
+
+import ldbc.snb.bteronhplus.tools.FileTools;
+import org.apache.hadoop.conf.Configuration;
+import org.python.antlr.ast.arguments;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+
+public class GraphStats {
+
+ private HashMap degreeDistribution = null;
+ private HashMap ccPerDegree = null;
+ private EmpiricalDistribution communitySizeDistribution = null;
+ private HashMap communityDensityMap = null;
+ private Set communitySizes = null;
+
+ public GraphStats(String degreeSequenceFile,
+ String ccPerDegreeSequenceFile,
+ String communitySizeSequenceFile,
+ String communityDensityMapFile
+ ) {
+
+
+ Configuration conf = new Configuration();
+ degreeDistribution = new HashMap();
+ ccPerDegree = new HashMap();
+ communitySizeDistribution = new EmpiricalDistribution(communitySizeSequenceFile, conf, 0);
+ communityDensityMap = new HashMap();
+ communitySizes = new HashSet();
+
+ // Loading degree distribution per community size
+ try {
+ BufferedReader reader = FileTools.getFile(degreeSequenceFile, conf);
+ String line;
+ line = reader.readLine();
+ while (line != null) {
+ String[] numbers = line.split(" ");
+ int degree = (int)Double.parseDouble(numbers[0]);
+ ArrayList observedSequence = new ArrayList();
+ for (int i = 1; i < numbers.length; ++i) {
+ observedSequence.add(Double.parseDouble(numbers[i]));
+ }
+
+ degreeDistribution.put(degree, new EmpiricalDistribution(observedSequence));
+ line = reader.readLine();
+ }
+ } catch (IOException e ) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ // Loading clustering coefficient per degree distributions
+ try {
+ BufferedReader reader = FileTools.getFile(ccPerDegreeSequenceFile, conf);
+ String line;
+ line = reader.readLine();
+ while (line != null) {
+ String[] numbers = line.split(" ");
+ int degree = (int)Double.parseDouble(numbers[0]);
+ ArrayList observedSequence = new ArrayList();
+ for (int i = 1; i < numbers.length; ++i) {
+ observedSequence.add(Double.parseDouble(numbers[i]));
+ }
+
+ if(observedSequence.size() == 1) {
+ observedSequence.add(observedSequence.get(0));
+ }
+ ccPerDegree.put(degree, new EmpiricalDistribution(observedSequence));
+ line = reader.readLine();
+ }
+ } catch (IOException e ) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+
+ // Community density map file
+ try {
+ BufferedReader reader = FileTools.getFile(communityDensityMapFile, conf);
+ String line;
+ line = reader.readLine();
+ while (line != null) {
+ String[] numbers = line.split(" ");
+ int degree = (int)Double.parseDouble(numbers[0]);
+ double density = (double)Double.parseDouble(numbers[1]);
+ communityDensityMap.put(degree,density);
+ line = reader.readLine();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ // Loading community size distribution
+ try {
+ BufferedReader reader = FileTools.getFile(communitySizeSequenceFile, conf);
+ String line;
+ line = reader.readLine();
+ while (line != null) {
+ communitySizes.add((int)(double)Double.parseDouble(line));
+ line = reader.readLine();
+ }
+ } catch (IOException e ) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ }
+
+ public EmpiricalDistribution getDegreeDistribution(Integer communitySize) {
+ return degreeDistribution.get(communitySize);
+ }
+
+ public EmpiricalDistribution getCommunitySizeDistribution() {
+ return communitySizeDistribution;
+ }
+
+ public EmpiricalDistribution getCCPerDegreeDistribution(int degree) {
+ return ccPerDegree.get(degree);
+ }
+
+ public double getCommunityDensity(int size) {
+ Double density = communityDensityMap.get(size);
+ if(density == null) {
+ for (int i = size-1; i >= 0; --i) {
+ density = communityDensityMap.get(i);
+ if(density != null) {
+ communityDensityMap.put(size,density);
+ break;
+ }
+ }
+ }
+ return density;
+ }
+
+ public Set getCommunitySizes() {
+ return communitySizes;
+ }
+}
diff --git a/src/main/java/ldbc/snb/bteronh/structures/Arguments.java b/src/main/java/ldbc/snb/bteronhplus/structures/Properties.java
similarity index 95%
rename from src/main/java/ldbc/snb/bteronh/structures/Arguments.java
rename to src/main/java/ldbc/snb/bteronhplus/structures/Properties.java
index cfe0dd8..bf96340 100644
--- a/src/main/java/ldbc/snb/bteronh/structures/Arguments.java
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/Properties.java
@@ -1,4 +1,4 @@
-package ldbc.snb.bteronh.structures;
+package ldbc.snb.bteronhplus.structures;
import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.Parameter;
@@ -9,7 +9,7 @@
/**
* Created by aprat on 17/08/16.
*/
-public class Arguments {
+public class Properties {
public static class Property {
private String property;
private String value;
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/RandomVariateStreamer.java b/src/main/java/ldbc/snb/bteronhplus/structures/RandomVariateStreamer.java
new file mode 100644
index 0000000..51b0473
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/RandomVariateStreamer.java
@@ -0,0 +1,70 @@
+package ldbc.snb.bteronhplus.structures;
+
+import umontreal.iro.lecuyer.randvar.RandomVariateGen;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Random;
+
+public class RandomVariateStreamer implements CommunityStreamer {
+
+ public static class RandomNode implements SuperNode {
+
+ private int degree;
+ private int id;
+
+ RandomNode(int id, int degree) {
+ this.id = id;
+ this.degree = degree;
+ }
+
+ @Override
+ public long getSize() {
+ return 1;
+ }
+
+ @Override
+ public long getInternalDegree() {
+ return 0;
+ }
+
+ @Override
+ public long getExternalDegree() {
+ return degree;
+ }
+
+ @Override
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public void sampleEdges(FileWriter writer, Random random, long numEdges, long offset) throws IOException {
+ }
+
+ @Override
+ public long sampleNode(Random random, long offset) {
+ return 0;
+ }
+
+ }
+
+ RandomVariateGen randomVariateGen;
+ int nextId = 0;
+
+ public RandomVariateStreamer(RandomVariateGen randomVariateGen) {
+ this.randomVariateGen = randomVariateGen;
+ }
+
+
+ @Override
+ public Community getModel(int id) {
+ return null;
+ }
+
+ @Override
+ public Community next() {
+ return null;
+ //return new RandomNode(nextId++, (int)(double)randomVariateGen.nextDouble());
+ }
+}
diff --git a/src/main/java/ldbc/snb/bteronhplus/structures/RealCommunityStreamer.java b/src/main/java/ldbc/snb/bteronhplus/structures/RealCommunityStreamer.java
new file mode 100644
index 0000000..9feb635
--- /dev/null
+++ b/src/main/java/ldbc/snb/bteronhplus/structures/RealCommunityStreamer.java
@@ -0,0 +1,98 @@
+package ldbc.snb.bteronhplus.structures;
+
+import ldbc.snb.bteronhplus.tools.FileTools;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.util.*;
+
+public class RealCommunityStreamer implements CommunityStreamer {
+
+ private ArrayList models;
+ private GraphStats stats;
+ private Random random;
+
+ public RealCommunityStreamer(GraphStats stats,
+ String communitiesFile,
+ Random random) {
+ this.random = random;
+ this.stats = stats;
+ Configuration conf = new Configuration();
+ this.models = new ArrayList();
+ long totalObservedEdges = 0;
+ long totalExcessDegree = 0;
+ long totalObservedNodes = 0;
+ try {
+
+ int counter = 0;
+ ArrayList communitySizes = new ArrayList();
+ BufferedReader reader = FileTools.getFile(communitiesFile, conf);
+ String line = reader.readLine();
+ while (line != null) {
+ HashMap idMap = new HashMap();
+ ArrayList localDegrees = new ArrayList();
+ ArrayList clusteringCoefficient = new ArrayList();
+ ArrayList edges = new ArrayList();
+ String[] community = line.split("\\|");
+ String[] nodesstr = community[0].split(" ");
+ String[] edgesstr = community[1].split(" ");
+ for(int i = 0; i < nodesstr.length; ++i) {
+ String nodeInfo[] = nodesstr[i].split(":");
+ idMap.put(Integer.parseInt(nodeInfo[0]), i);
+ localDegrees.add(Integer.parseInt(nodeInfo[1]));
+ clusteringCoefficient.add(Double.parseDouble(nodeInfo[2]));
+ }
+
+
+ Map