CDK-476 oozie UriHandler for Kite URIs

kite-data/kite-data-oozie/pom.xml

@@ -0,0 +1,194 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+~ Copyright 2015 Cloudera Inc.
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>kite-data-oozie</artifactId>
+
+  <parent>
+    <groupId>org.kitesdk</groupId>
+    <artifactId>kite-data</artifactId>
+    <version>1.0.1-SNAPSHOT</version>
+  </parent>
+
+  <name>Kite Data Oozie Module</name>
+  <description>
+    The Kite Data Oozie module provides Oozie support for working with Kite datasets.
+  </description>
+
+  <properties>
+    <vers.oozie4>4.1.0</vers.oozie4>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <configuration>
+          <excludePackageNames>org.kitesdk.data.oozie</excludePackageNames>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>findbugs-maven-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>compile</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <tasks>
+                <echo message="Create empty javadoc JAR to satisfy Maven central" />
+                <mkdir dir="target/apidocs" />
+              </tasks>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+  <reporting>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-project-info-reports-plugin</artifactId>
+        <reportSets>
+          <reportSet>
+            <inherited>false</inherited>
+            <reports>
+              <report>index</report>
+              <report>summary</report>
+              <report>dependency-info</report>
+              <report>dependencies</report>
+            </reports>
+          </reportSet>
+        </reportSets>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </reporting>
+
+  <dependencies>
+    <!-- Kite -->
+    <dependency>
+      <groupId>org.kitesdk</groupId>
+      <artifactId>kite-data-core</artifactId>
+      <version>${project.parent.version}</version>
+    </dependency>
+
+    <!-- Hadoop -->
+    <dependency>
+      <groupId>org.kitesdk</groupId>
+      <artifactId>${artifact.hadoop-deps}</artifactId>
+      <type>pom</type>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <!-- conflicts with jetty included with HBase -->
+          <groupId>tomcat</groupId>
+          <artifactId>jasper-runtime</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.kitesdk</groupId>
+      <artifactId>kite-hadoop-compatibility</artifactId>
+    </dependency>
+
+    <!-- custom URI handlers require Oozie 4.x -->
+    <dependency>
+      <groupId>org.apache.oozie</groupId>
+      <artifactId>oozie-core</artifactId>
+      <version>${vers.oozie4}</version>
+    </dependency>
+
+    <!-- Misc -->
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+      <scope>provided</scope>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>annotations</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <!-- Test Dependencies -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.kitesdk</groupId>
+      <artifactId>kite-data-core</artifactId>
+      <version>${project.parent.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.kitesdk</groupId>
+      <artifactId>${artifact.hadoop-test-deps}</artifactId>
+      <type>pom</type>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <profiles>
+    <profile>
+      <id>cdh5</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>cdh5</value>
+        </property>
+      </activation>
+      <properties>
+        <vers.oozie4>${vers.oozie}</vers.oozie4>
+      </properties>
+    </profile>
+  </profiles>
+
+</project>

kite-data/kite-data-oozie/src/main/java/org/kitesdk/data/oozie/KiteLauncherURIHandler.java

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2015 Cloudera Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kitesdk.data.oozie;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.action.hadoop.LauncherException;
+import org.apache.oozie.action.hadoop.LauncherURIHandler;
+
+public class KiteLauncherURIHandler implements LauncherURIHandler {
+
+  @Override
+  public boolean create(final URI uri, final Configuration conf)
+      throws LauncherException {
+    throw new UnsupportedOperationException(
+        "Creation of resources is not supported for Kite URIs.");
+  }
+
+  @Override
+  public boolean delete(final URI uri, final Configuration conf)
+      throws LauncherException {
+    throw new UnsupportedOperationException(
+        "Deletion of resources is not supported for Kite URIs.");
+  }
+
+  @Override
+  public List<Class<?>> getClassesForLauncher() {
+    return Collections.emptyList();
+  }
+
+}

kite-data/kite-data-oozie/src/main/java/org/kitesdk/data/oozie/KiteURIHandler.java

@@ -0,0 +1,157 @@
+/**
+ * Copyright 2015 Cloudera Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kitesdk.data.oozie;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.ErrorCode;
+import org.apache.oozie.XException;
+import org.apache.oozie.action.hadoop.LauncherURIHandler;
+import org.apache.oozie.dependency.URIHandler;
+import org.apache.oozie.dependency.URIHandlerException;
+import org.apache.oozie.service.HadoopAccessorException;
+import org.apache.oozie.service.URIHandlerService;
+import org.apache.oozie.util.XLog;
+import org.kitesdk.data.DatasetException;
+import org.kitesdk.data.DatasetNotFoundException;
+import org.kitesdk.data.Datasets;
+import org.kitesdk.data.Signalable;
+import org.kitesdk.data.URIBuilder;
+import org.kitesdk.data.View;
+
+/**
+ * A Kite URI handler that works with {@link Signalable} views.
+ *
+ * To be considered as {@link #exists(URI, Configuration, String) existing} the
+ * view must have been signaled as ready.
+ */
+public class KiteURIHandler implements URIHandler {
+
+  private static final XLog LOG = XLog.getLog(KiteURIHandler.class);
+
+  private Set<String> supportedSchemes;
+  private List<Class<?>> classesToShip;
+
+  @Override
+  public void init(final Configuration conf) {
+    supportedSchemes = new HashSet<String>();
+    final String[] schemes = conf.getStrings(
+        URIHandlerService.URI_HANDLER_SUPPORTED_SCHEMES_PREFIX
+            + this.getClass().getSimpleName()
+            + URIHandlerService.URI_HANDLER_SUPPORTED_SCHEMES_SUFFIX, "view", "dataset");
+    supportedSchemes.addAll(Arrays.asList(schemes));
+    classesToShip = new KiteLauncherURIHandler().getClassesForLauncher();
+  }
+
+  @Override
+  public Set<String> getSupportedSchemes() {
+    return supportedSchemes;
+  }
+
+  @Override
+  public Class<? extends LauncherURIHandler> getLauncherURIHandlerClass() {
+    return KiteLauncherURIHandler.class;
+  }
+
+  @Override
+  public List<Class<?>> getClassesForLauncher() {
+    return classesToShip;
+  }
+
+  @Override
+  public DependencyType getDependencyType(final URI uri)
+      throws URIHandlerException {
+    return DependencyType.PULL;
+  }
+
+  @Override
+  public void registerForNotification(final URI uri, final Configuration conf,
+      final String user, final String actionID) throws URIHandlerException {
+    throw new UnsupportedOperationException(
+        "Notifications are not supported for " + uri);
+  }
+
+  @Override
+  public boolean unregisterFromNotification(final URI uri, final String actionID) {
+    throw new UnsupportedOperationException(
+        "Notifications are not supported for " + uri);
+  }
+
+  @Override
+  public Context getContext(final URI uri, final Configuration conf,
+      final String user) throws URIHandlerException {
+    return null;
+  }
+
+  @SuppressWarnings("rawtypes")
+  @Override
+  public boolean exists(final URI uri, final Context context) throws URIHandlerException {
+    try {
+      View<GenericRecord> view = Datasets.load(uri);
+      if(view instanceof Signalable) {
+        return ((Signalable)view).isReady();
+      }
+    } catch (IllegalArgumentException ex) {
+      LOG.error("the URI " + uri + " was not a view or dataset");
+    } catch (DatasetNotFoundException ex) {
+      LOG.error("the dataset for the URI "+ uri +" did not actually exist");
+    } catch (DatasetException e) {
+      throw new HadoopAccessorException(ErrorCode.E0902, e);
+    }
+    // didn't meet all the requirements for a URI/view that we want to use with oozie
+    return false;
+  }
+
+  @Override
+  public boolean exists(final URI uri, final Configuration conf, final String user)
+      throws URIHandlerException {
+    // currently does not handle access limitations between the oozie user and
+    // a potential dataset owner
+    return exists(uri, null);
+  }
+
+  @Override
+  public String getURIWithDoneFlag(final String uri, final String doneFlag)
+      throws URIHandlerException {
+    return uri;
+  }
+
+  @Override
+  public void validate(final String uri) throws URIHandlerException {
+    try {
+      URI uriParsed = URI.create(uri);
+      String scheme = uriParsed.getScheme();
+      if(!(URIBuilder.VIEW_SCHEME.equals(scheme) || URIBuilder.DATASET_SCHEME.equals(scheme))) {
+        LOG.error("Unexpected scheme: view, uri was "+ uri);
+        XException xException = new XException(ErrorCode.E0904, scheme, uri);
+        throw new URIHandlerException(xException);
+      }
+    } catch (IllegalArgumentException iae) {
+      XException xException = new XException(ErrorCode.E0906, iae);
+      throw new URIHandlerException(xException);
+    }
+  }
+
+  @Override
+  public void destroy() {
+  }
+
+}