From 7dcf89af25c2f67209662ebaac073193e6407399 Mon Sep 17 00:00:00 2001 From: "E. Sammer" Date: Sun, 28 Aug 2016 13:34:26 -0700 Subject: [PATCH 1/2] Added classes for generating event IDs. - Includes the EventIdGenerator interface and implementations in the new package com.osso.event.id. - Javadoc exists for all classes. - Contains the addition of Guava 15. It may make sense to shade this in a follow up commit. --- pom.xml | 6 + .../com/osso/event/id/EventIdGenerator.java | 48 ++++++++ .../com/osso/event/id/EventIdGeneratorV1.java | 102 +++++++++++++++++ .../com/osso/event/id/EventIdGeneratorV3.java | 103 ++++++++++++++++++ src/main/java/com/osso/event/id/EventIds.java | 59 ++++++++++ .../java/com/osso/event/id/package-info.java | 21 ++++ .../osso/event/id/TestEventIdGeneratorV1.java | 57 ++++++++++ .../osso/event/id/TestEventIdGeneratorV3.java | 54 +++++++++ .../java/com/osso/event/id/TestEventIds.java | 54 +++++++++ 9 files changed, 504 insertions(+) create mode 100644 src/main/java/com/osso/event/id/EventIdGenerator.java create mode 100644 src/main/java/com/osso/event/id/EventIdGeneratorV1.java create mode 100644 src/main/java/com/osso/event/id/EventIdGeneratorV3.java create mode 100644 src/main/java/com/osso/event/id/EventIds.java create mode 100644 src/main/java/com/osso/event/id/package-info.java create mode 100644 src/test/java/com/osso/event/id/TestEventIdGeneratorV1.java create mode 100644 src/test/java/com/osso/event/id/TestEventIdGeneratorV3.java create mode 100644 src/test/java/com/osso/event/id/TestEventIds.java diff --git a/pom.xml b/pom.xml index 697572e..d6104d6 100644 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,7 @@ 1.7.6 + 15.0 4.12 @@ -236,6 +237,11 @@ + + com.google.guava + guava + ${version.guava} + org.apache.avro avro diff --git a/src/main/java/com/osso/event/id/EventIdGenerator.java b/src/main/java/com/osso/event/id/EventIdGenerator.java new file mode 100644 index 0000000..954d427 --- /dev/null +++ b/src/main/java/com/osso/event/id/EventIdGenerator.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016 Rocana. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.osso.event.id; + +import com.osso.event.Event; + +/** + *

+ * A generator of {@link Event} IDs. + *

+ *

+ * Implementations of this interface generate event IDs according to a specific + * version of the Osso specification. While the exact definition of how event + * IDs are generated is defined by the specification, all implementations make + * the following guarantees. + *

+ * Most implementations are hashing strategies that include a set of the event + * fields to generate a unique event ID, save for the rare hash collisions. + *

+ * + * @see EventIds + */ +public interface EventIdGenerator { + + String generateId(Event event); + +} diff --git a/src/main/java/com/osso/event/id/EventIdGeneratorV1.java b/src/main/java/com/osso/event/id/EventIdGeneratorV1.java new file mode 100644 index 0000000..be5b69f --- /dev/null +++ b/src/main/java/com/osso/event/id/EventIdGeneratorV1.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2016 Rocana. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.osso.event.id; + +import com.google.common.base.Charsets; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; +import com.osso.event.Event; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + *

+ * The event ID generation implementation for version 1 of the Osso + * specification. + *

+ *

+ * This implementation is a base 32 encoding of a SHA-256 hash of the + * following fields, in the following order: + *

+ *

+ *

+ * The exact implementations of the base 32 encoding and SHA-256 hash function + * are Google Guava's {@link BaseEncoding#base32()} and + * {@link Hashing#sha256()}, respectively. The base 32 encoding is + * RFC-4648 section 6 + * compliant, uses uppercase characters, and the '=' character for + * padding. Guava's SHA-256 implementation is a wrapper around Java's + * {@link java.security.MessageDigest}.Integers and longs are encoded in + * little endian format, while strings use Java's + * {@link String#getBytes(java.nio.charset.Charset)} with a UTF-8 character + * set, which uses the appropriate {@link java.nio.charset.CharsetEncoder}. + *

+ */ +public final class EventIdGeneratorV1 implements EventIdGenerator { + + private static final BaseEncoding baseEncoding = BaseEncoding.base32(); + private static final HashFunction hashFunction = Hashing.sha256(); + + @Override + public String generateId(Event event) { + Hasher hasher = hashFunction.newHasher(); + + Map attributes = event.getAttributes(); + + List attrKeys = new ArrayList<>(attributes.keySet()); + Collections.sort(attrKeys); + + for (String attrKey : attrKeys) { + hasher.putString(attrKey, Charsets.UTF_8) + .putString(attributes.get(attrKey), Charsets.UTF_8); + } + + hasher.putString(String.valueOf(event.getTs()), Charsets.UTF_8); + + if (event.getSource() != null) { + hasher.putString(event.getSource(), Charsets.UTF_8); + } + + hasher.putString(event.getLocation(), Charsets.UTF_8) + .putString(event.getHost(), Charsets.UTF_8) + .putString(event.getService(), Charsets.UTF_8) + .putString(String.valueOf(event.getEventTypeId()), Charsets.UTF_8); + + if (event.getBody() != null) { + hasher.putBytes(event.getBody().array()); + } + + return baseEncoding.encode(hasher.hash().asBytes()); + } + +} diff --git a/src/main/java/com/osso/event/id/EventIdGeneratorV3.java b/src/main/java/com/osso/event/id/EventIdGeneratorV3.java new file mode 100644 index 0000000..81476b9 --- /dev/null +++ b/src/main/java/com/osso/event/id/EventIdGeneratorV3.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016 Rocana. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.osso.event.id; + +import com.google.common.base.Charsets; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; +import com.osso.event.Event; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + *

+ * The event ID generation implementation for version 3 of the Osso + * specification. + *

+ *

+ * This implementation is a base 32 encoding of a SHA-256 hash of the + * following fields, in the following order: + *

    + *
  • version
    • + *
  • ts
    • + *
  • event_type_id
    • + *
  • location
    • + *
  • host
    • + *
  • service
    • + *
  • source, if defined.
    • + *
  • body, if defined.
    • + *
  • + * Each attribute key, then value, in sorted order by the attribute key. + *
    • + *
+ *

+ *

+ * The exact implementations of the base 32 encoding and SHA-256 hash function + * are Google Guava's {@link BaseEncoding#base32()} and + * {@link Hashing#sha256()}, respectively. The base 32 encoding is + * RFC-4648 section 6 + * compliant, uses uppercase characters, and the '=' character for + * padding. Guava's SHA-256 implementation is a wrapper around Java's + * {@link java.security.MessageDigest}. Integers and longs are encoded in + * little endian format, while strings use Java's + * {@link String#getBytes(java.nio.charset.Charset)} with a UTF-8 character + * set, which uses the appropriate {@link java.nio.charset.CharsetEncoder}. + *

+ */ +public final class EventIdGeneratorV3 implements EventIdGenerator { + + private static final BaseEncoding baseEncoding = BaseEncoding.base32(); + private static final HashFunction hashFunction = Hashing.sha256(); + + @Override + public String generateId(Event event) { + Hasher hasher = hashFunction.newHasher(); + + hasher.putLong(event.getVersion()) + .putLong(event.getTs()) + .putLong(event.getEventTypeId()) + .putString(event.getLocation(), Charsets.UTF_8) + .putString(event.getHost(), Charsets.UTF_8) + .putString(event.getService(), Charsets.UTF_8); + + if (event.getSource() != null) { + hasher.putString(event.getSource(), Charsets.UTF_8); + } + + if (event.getBody() != null) { + hasher.putBytes(event.getBody().array()); + } + + Map attributes = event.getAttributes(); + + List attrKeys = new ArrayList<>(attributes.keySet()); + Collections.sort(attrKeys); + + for (String attrKey : attrKeys) { + hasher.putString(attrKey, Charsets.UTF_8) + .putString(attributes.get(attrKey), Charsets.UTF_8); + } + + return baseEncoding.encode(hasher.hash().asBytes()); + } + +} diff --git a/src/main/java/com/osso/event/id/EventIds.java b/src/main/java/com/osso/event/id/EventIds.java new file mode 100644 index 0000000..fde0499 --- /dev/null +++ b/src/main/java/com/osso/event/id/EventIds.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016 Rocana. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.osso.event.id; + +import com.osso.event.Event; +import org.apache.avro.Schema; + +public final class EventIds { + + private static final EventIdGenerator generatorV1 = new EventIdGeneratorV1(); + private static final EventIdGenerator generatorV3 = new EventIdGeneratorV3(); + + // Prevent instantiation. + private EventIds() { + } + + public static Event populateId(Event event) { + event.setId(generatorForEvent(event).generateId(event)); + + return event; + } + + public static EventIdGenerator generatorForEvent(Event event) { + Schema schema = event.getSchema(); + + /* + * Version field was added in v3. Its presence is enough to tell us we + * should use the v3 generator, at least for now. + */ + if (schema.getField("version") != null) { + return generatorV3(); + } else { + return generatorV1(); + } + } + + public static EventIdGenerator generatorV1() { + return generatorV1; + } + + public static EventIdGenerator generatorV3() { + return generatorV3; + } + +} diff --git a/src/main/java/com/osso/event/id/package-info.java b/src/main/java/com/osso/event/id/package-info.java new file mode 100644 index 0000000..73a345d --- /dev/null +++ b/src/main/java/com/osso/event/id/package-info.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2016 Rocana. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Support for generating Osso-compliant event IDs. + */ + +package com.osso.event.id; \ No newline at end of file diff --git a/src/test/java/com/osso/event/id/TestEventIdGeneratorV1.java b/src/test/java/com/osso/event/id/TestEventIdGeneratorV1.java new file mode 100644 index 0000000..9629a42 --- /dev/null +++ b/src/test/java/com/osso/event/id/TestEventIdGeneratorV1.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2016 Rocana. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.osso.event.id; + +import com.osso.event.Event; +import com.osso.event.id.EventIdGenerator; +import com.osso.event.id.EventIdGeneratorV1; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +public class TestEventIdGeneratorV1 { + + @Test + public void testGenerateId() { + EventIdGenerator generator = new EventIdGeneratorV1(); + + Map attributes = new HashMap<>(); + attributes.put("a", "1"); + attributes.put("b", "2"); + attributes.put("c", "3"); + + Event event = Event.newBuilder() + .setTs(0) + .setEventTypeId(1) + .setSource("source") + .setLocation("location") + .setHost("host") + .setService("service") + .setBody(ByteBuffer.wrap("String body".getBytes())) + .setAttributes(attributes) + .build(); + + Assert.assertEquals( + "DJMAWB6VOIBYGKQZ4E2XXBVMIZW2CXTM3NPWFJW32DYJGMA47IPQ====", + generator.generateId(event) + ); + } + +} \ No newline at end of file diff --git a/src/test/java/com/osso/event/id/TestEventIdGeneratorV3.java b/src/test/java/com/osso/event/id/TestEventIdGeneratorV3.java new file mode 100644 index 0000000..aaeb5ae --- /dev/null +++ b/src/test/java/com/osso/event/id/TestEventIdGeneratorV3.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016 Rocana. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.osso.event.id; + +import com.osso.event.Event; +import com.osso.event.id.EventIdGenerator; +import com.osso.event.id.EventIdGeneratorV3; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +public class TestEventIdGeneratorV3 { + + @Test + public void testGenerateId() { + EventIdGenerator generator = new EventIdGeneratorV3(); + + Map attributes = new HashMap<>(); + attributes.put("a", "1"); + attributes.put("b", "2"); + attributes.put("c", "3"); + + Event event = Event.newBuilder() + .setTs(0) + .setEventTypeId(1) + .setSource("source") + .setLocation("location") + .setHost("host") + .setService("service") + .setBody(ByteBuffer.wrap("String value".getBytes())) + .setAttributes(attributes) + .build(); + + Assert.assertEquals("SX7F5J4XHODRJOJTMNFBUA2NLGDNPRAY5QW36EK4ZLKB5RWHVXPQ====", generator.generateId(event)); + } + +} \ No newline at end of file diff --git a/src/test/java/com/osso/event/id/TestEventIds.java b/src/test/java/com/osso/event/id/TestEventIds.java new file mode 100644 index 0000000..f443c0a --- /dev/null +++ b/src/test/java/com/osso/event/id/TestEventIds.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016 Rocana. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.osso.event.id; + +import com.osso.event.Event; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; + +public class TestEventIds { + + @Test + public void testV1() { + EventIdGenerator generator = EventIds.generatorV1(); + + Assert.assertNotNull(generator); + } + + @Test + public void testV3() { + EventIdGenerator generator = EventIds.generatorV3(); + + Assert.assertNotNull(generator); + } + + @Test + public void testPopulateEvent() { + Event event = Event.newBuilder() + .setTs(0) + .setEventTypeId(1) + .setAttributes(new HashMap()) + .build(); + + EventIds.populateId(event); + + Assert.assertEquals("O4DEUDH36ZLHLF54VDEH5JUOJX67DPK5AVR4WJ7QS34QWO2NX7VA====", event.getId()); + } + +} \ No newline at end of file From cf089550255167c2314f6fb90d8f9cd02a805e57 Mon Sep 17 00:00:00 2001 From: "E. Sammer" Date: Sun, 28 Aug 2016 13:44:13 -0700 Subject: [PATCH 2/2] Added missing docs for EventIds. - These docs were lost in a bad `git reset` and should be have included in the previous commit. --- src/main/java/com/osso/event/id/EventIds.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/java/com/osso/event/id/EventIds.java b/src/main/java/com/osso/event/id/EventIds.java index fde0499..7b7586b 100644 --- a/src/main/java/com/osso/event/id/EventIds.java +++ b/src/main/java/com/osso/event/id/EventIds.java @@ -19,6 +19,9 @@ import com.osso.event.Event; import org.apache.avro.Schema; +/** + * Convenience functions for working with event IDs. + */ public final class EventIds { private static final EventIdGenerator generatorV1 = new EventIdGeneratorV1(); @@ -28,12 +31,33 @@ public final class EventIds { private EventIds() { } + /** + *

+ * Populate the given event's id field using an ID generator most + * appropriate for the event's version. + *

+ *

+ * As its name implies, this method mutates the supplied event. If that's + * not what you want, use {@link #generatorForEvent(Event)} and use it + * to generate an ID directly. + *

+ * + * @param event The event for which an ID should be generated and populated, + * and from which the version will be determined. + * @return A reference to the given event to make call chaining easier. + */ public static Event populateId(Event event) { event.setId(generatorForEvent(event).generateId(event)); return event; } + /** + * Get an event ID generator most appropriate for the event's version. + * + * @param event The event from which the version will be determined. + * @return An implementation of {@link EventIdGenerator}. + */ public static EventIdGenerator generatorForEvent(Event event) { Schema schema = event.getSchema();