0
public static void main(String[] args) {
//Pipeline p = Pipeline.create(PipelineOptionsFactory.fromArgs(args).withValidation().create());
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setStagingLocation("gs://bucketname/stageapache");
options.setTempLocation("gs://bucketname/stageapachetemp");
options.setProject("projectid");
Pipeline p=Pipeline.create(options);
p.apply(TextIO.read().from("gs://bucketname/filename.csv"));
//p.apply(FileIO.match().filepattern("gs://bucketname/f.csv"));
p.run();
}
のpom.xmlアパッチビーム2.2 pipeline.applyそのような方法の例外
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-io-google-cloud-platform</artifactId>
<version>2.0.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.beam/beam-runners-google-cloud-dataflow-java -->
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
<version>2.0.0</version>
</dependency>
エラー
Dec 08, 2017 5:09:35 PM org.apache.beam.runners.dataflow.DataflowRunner fromOptions
INFO: PipelineOptions.filesToStage was not specified. Defaulting to files from the classpath: will stage 85 files. Enable logging at DEBUG level to see which files will be staged.
Exception in thread "main" java.lang.NoSuchMethodError: org.apache.beam.sdk.values.PCollection.createPrimitiveOutputInternal(Lorg/apache/beam/sdk/Pipeline;Lorg/apache/beam/sdk/values/WindowingStrategy;Lorg/apache/beam/sdk/values/PCollection$IsBounded;)Lorg/apache/beam/sdk/values/PCollection;
at org.apache.beam.runners.dataflow.PrimitiveParDoSingleFactory$ParDoSingle.expand(PrimitiveParDoSingleFactory.java:68)
at org.apache.beam.runners.dataflow.PrimitiveParDoSingleFactory$ParDoSingle.expand(PrimitiveParDoSingleFactory.java:58)
at org.apache.beam.sdk.Pipeline.applyReplacement(Pipeline.java:550)
at org.apache.beam.sdk.Pipeline.replace(Pipeline.java:280)
at org.apache.beam.sdk.Pipeline.replaceAll(Pipeline.java:201)
at org.apache.beam.runners.dataflow.DataflowRunner.replaceTransforms(DataflowRunner.java:688)
at org.apache.beam.runners.dataflow.DataflowRunner.run(DataflowRunner.java:498)
at org.apache.beam.runners.dataflow.DataflowRunner.run(DataflowRunner.java:153)
at org.apache.beam.sdk.Pipeline.run(Pipeline.java:303)
at org.apache.beam.sdk.Pipeline.run(Pipeline.java:289)
at com.pearson.apachebeam.StarterPipeline.main(StarterPipeline.java:60)
iが上記のエラーを取得していFILEIO/TextIOラインを追加する場合私はそれを実行している場合、その行を追加すると、そこには操作がないので失敗しているジョブを作成しています。私は、私は
ヘルプは問題があなたのpom.xml
が依存していることである
おかげ
ありがとう、たくさんのjkff。私はFileIOを使って読むことができます – Damodar