polis-vk · Georgiy0 · Nov 24, 2017 · Nov 24, 2017 · Nov 24, 2017 · sergboec
diff --git a/2017-big-data.iml b/2017-big-data.iml
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-client:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-common:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: com.google.guava:guava:11.0.2" level="project" />
+    <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.1.1" level="project" />
+    <orderEntry type="library" name="Maven: xmlenc:xmlenc:0.52" level="project" />
+    <orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />
+    <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.4" level="project" />
+    <orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
+    <orderEntry type="library" name="Maven: commons-net:commons-net:3.1" level="project" />
+    <orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.1" level="project" />
+    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />
+    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
+    <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
+    <orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />
+    <orderEntry type="library" name="Maven: commons-digester:commons-digester:1.8" level="project" />
+    <orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.7.0" level="project" />
+    <orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.5" level="project" />
+    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.5" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.9.13" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.9.13" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.4" level="project" />
+    <orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.3" level="project" />
+    <orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" level="project" />
+    <orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" />
+    <orderEntry type="library" name="Maven: com.google.code.gson:gson:2.2.4" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.2.4" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.curator:curator-client:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.curator:curator-recipes:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
+    <orderEntry type="library" name="Maven: org.htrace:htrace-core:3.0.4" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
+    <orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-hdfs:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.26" level="project" />
+    <orderEntry type="library" name="Maven: io.netty:netty:3.6.2.Final" level="project" />
+    <orderEntry type="library" name="Maven: xerces:xercesImpl:2.9.1" level="project" />
+    <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.3.04" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" />
+    <orderEntry type="library" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" />
+    <orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
+    <orderEntry type="library" name="Maven: javax.servlet:servlet-api:2.5" level="project" />
+    <orderEntry type="library" name="Maven: com.sun.jersey:jersey-core:1.9" level="project" />
+    <orderEntry type="library" name="Maven: com.sun.jersey:jersey-client:1.9" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.9.13" level="project" />
+    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-xc:1.9.13" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.6.0" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-annotations:2.6.0" level="project" />
+  </component>
+</module>
diff --git a/README.md b/README.md
@@ -1,27 +1,7 @@
 # 2017-big-data
 Репозиторий для практических домашних заданий 2017 года курса "Введение в машинное обучение для java-разработчиков" в [Технополис](https://polis.mail.ru).
 
-### Fork
-[Форкните проект](https://help.github.com/articles/fork-a-repo/), склонируйте и добавьте `upstream`:
-```
-$ git clone [email protected]:<username>/2017-big-data.git
-Cloning into '2017-big-data'...
-remote: Counting objects: 3, done.
-remote: Compressing objects: 100% (2/2), done.
-remote: Total 3 (delta 2), reused 3 (delta 2), pack-reused 0
-Receiving objects: 100% (3/3), 10.1 KiB | 4.5 MiB/s, done.
-Resolving deltas: 100% (2/2), done.
-$ git remote add upstream [email protected]:polis-mail-ru/2017-big-data.git
-$ git fetch upstream
-From github.com:polis-mail-ru/2017-big-data
- * [new branch]      master     -> upstream/master
-```
+### Job run time
 
-### Develop
-Откройте в IDE -- [IntelliJ IDEA Community Edition](https://www.jetbrains.com/idea/) 
-через File->Open, выбрав pom.xml
-
-Пример конфигурация для запуска таски из idea можно посмотреть в файле config.png
-
-В своём Java package `<username>` выполните все задания.
+Время работы job WordCount без combiner составило 47m 38.473s. С использованием combiner, после добавления combiner время работы составило 31m 38.491s.
 
diff --git a/pom.xml b/pom.xml
@@ -29,7 +29,7 @@
         <configuration>
           <archive>
             <manifest>
-              <mainClass>pritykovskaya.WordCount</mainClass>
+              <mainClass>kubrin.WordCount</mainClass>
             </manifest>
           </archive>
         </configuration>
@@ -38,8 +38,8 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
-          <source>1.6</source>
-          <target>1.6</target>
+          <source>1.8</source>
+          <target>1.8</target>
         </configuration>
       </plugin>
     </plugins>

diff --git a/runtime.txt b/runtime.txt
@@ -0,0 +1,18 @@
+WordCount without combiner
+real	47m38.473s
+user	44m32.444s
+sys	1m5.972s
+
+
+WordCount with combiner
+real	31m38.491s
+user	33m37.704s
+sys	0m19.676s
+
+WordsSort
+real	0m23.659s
+user	0m25.428s
+sys	0m1.880s
+
+
+
diff --git a/src/main/java/META-INF/MANIFEST.MF b/src/main/java/META-INF/MANIFEST.MF
@@ -0,0 +1,3 @@
+Manifest-Version: 1.0
+Main-Class: kubrin.SortWords
+
diff --git a/src/main/java/kubrin/GetRow.java b/src/main/java/kubrin/GetRow.java
@@ -0,0 +1,118 @@
+package kubrin;
+// based on implementation of a-filippo (https://github.com/a-filippo/2017-big-data)
+
+import java.io.IOException;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+public class GetRow extends Configured implements Tool {
+    private static int valueNumber;
+    private static IntWritable ONE = new IntWritable(1);
+
+    static class MyMapper extends Mapper<Object, Text, IntWritable, TextWithCountWritable>{
+        private int count;
+
+        @Override
+        protected void setup(Context context) throws IOException, InterruptedException {
+            super.setup(context);
+            count = 0;
+        }
+
+        @Override
+        protected void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+            final String line = value.toString();
+
+            int pos = line.indexOf(0x09);
+
+            int inputCount = Integer.valueOf(line.substring(0, pos));
+            String inputString = line.substring(pos+1);
+
+            if (count <= valueNumber){
+                context.write(ONE, new TextWithCountWritable(inputString, inputCount));
+                count++;
+            }
+        }
+    }
+
+    static class MyReducer extends Reducer<IntWritable, TextWithCountWritable, Text, IntWritable>{
+        private SortedSet<TextWithCountWritable> setOfTextWithCount;
+
+        @Override
+        protected void setup(Context context) throws IOException, InterruptedException {
+            super.setup(context);
+            setOfTextWithCount = new TreeSet<>();
+        }
+
+        @Override
+        protected void reduce(IntWritable key, Iterable<TextWithCountWritable> values, Context context) throws IOException, InterruptedException {
+            values.forEach((textWithCountWriteble -> {
+                setOfTextWithCount.add(textWithCountWriteble.clone());
+                if (setOfTextWithCount.size() > valueNumber + 1){
+                    setOfTextWithCount.remove(setOfTextWithCount.last());
+                }
+            }));
+        }
+
+        @Override
+        protected void cleanup(Context context) throws IOException, InterruptedException {
+            super.cleanup(context);
+
+            int i = 0;
+            for (TextWithCountWritable textWithCountWriteble : setOfTextWithCount){
+                if (i == valueNumber){
+                    context.write(new Text(textWithCountWriteble.getText()), new IntWritable(textWithCountWriteble.getCount()));
+                    break;
+                }
+                i++;
+            }
+        }
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+        final Configuration conf = this.getConf();
+        final Job job = new Job(conf, "GetRow");
+
+        valueNumber = Integer.valueOf(args[2]);
+
+        job.setJarByClass(GetRow.class);
+
+        job.setInputFormatClass(TextInputFormat.class);
+        job.setOutputFormatClass(TextOutputFormat.class);
+
+        job.setMapperClass(MyMapper.class);
+        job.setReducerClass(MyReducer.class);
+
+        job.setMapOutputKeyClass(IntWritable.class);
+        job.setMapOutputValueClass(TextWithCountWritable.class);
+
+        job.setOutputKeyClass(Text.class);
+        job.setOutputValueClass(IntWritable.class);
+
+        FileInputFormat.addInputPath(job, new Path(args[0]));
+        FileOutputFormat.setOutputPath(job, new Path(args[1]));
+
+        return job.waitForCompletion(true) ? 0 : 1;
+    }
+
+    public static void main(String[] args) throws Exception{
+        final int returnCode = ToolRunner.run(new Configuration(), new GetRow(), args);
+        System.exit(returnCode);
+
+    }
+}
diff --git a/src/main/java/kubrin/NameCount.java b/src/main/java/kubrin/NameCount.java
@@ -0,0 +1,100 @@
+package kubrin;
+// based on implementation of a-filippo (https://github.com/a-filippo/2017-big-data)
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+public class NameCount extends Configured implements Tool {
+    private static final Pattern namePattern = Pattern.compile("^[A-Z][a-z0-9]*$");
+
+    static class MyMapper extends Mapper<Object, Text, Text, TextWithCountWritable>{
+
+        @Override
+        protected void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+            final String line = value.toString();
+
+            int pos = line.indexOf('\t');
+
+            String inputString = line.substring(0, pos);
+            int inputCount = Integer.valueOf(line.substring(pos+1));
+
+            context.write(new Text(inputString.toLowerCase()), new TextWithCountWritable(inputString, inputCount));
+        }
+    }
+
+    static class MyReducer extends Reducer<Text, TextWithCountWritable, Text, IntWritable>{
+
+        @Override
+        protected void reduce(Text key, Iterable<TextWithCountWritable> values, Context context) throws IOException, InterruptedException {
+            int sumAllForms = 0;
+            int rightFormCount = 0;
+            String rightFormText = null;
+
+            for (final TextWithCountWritable value : values){
+                sumAllForms += value.getCount();
+
+                if (rightFormText == null){
+                    Matcher matcher = namePattern.matcher(value.getText());
+                    if (matcher.matches()){
+                        rightFormText = value.getText();
+                        rightFormCount = value.getCount();
+                    }
+                }
+            }
+
+            if (rightFormText == null){
+                return;
+            }
+
+            if (rightFormCount / (double)sumAllForms >= 0.995){
+                context.write(new Text(rightFormText), new IntWritable(rightFormCount));
+            }
+        }
+    }
+
+    @Override
+    public int run(String[] args) throws Exception {
+        final Configuration conf = this.getConf();
+        final Job job = new Job(conf, "NamesPercent");
+        job.setJarByClass(NameCount.class);
+
+        job.setInputFormatClass(TextInputFormat.class);
+        job.setOutputFormatClass(TextOutputFormat.class);
+
+        job.setMapperClass(MyMapper.class);
+        job.setReducerClass(MyReducer.class);
+
+        job.setMapOutputKeyClass(Text.class);
+        job.setMapOutputValueClass(TextWithCountWritable.class);
+
+        job.setOutputKeyClass(Text.class);
+        job.setOutputValueClass(IntWritable.class);
+
+        FileInputFormat.addInputPath(job, new Path(args[0]));
+        FileOutputFormat.setOutputPath(job, new Path(args[1]));
+
+        return job.waitForCompletion(true) ? 0 : 1;
+    }
+
+    public static void main(String[] args) throws Exception{
+        final int returnCode = ToolRunner.run(new Configuration(), new NameCount(), args);
+        System.exit(returnCode);
+
+    }
+}