diff --git a/Makefile b/Makefile index 7d3f0be6..9c1cc211 100644 --- a/Makefile +++ b/Makefile @@ -31,4 +31,32 @@ test: @printf "\n run flake8::\n" && flake8 . @printf "\n run pylint::\n" && pylint --enable=E --disable=W,R,C --unsafe-load-any-extension=y examples/ naz/ tests/ cli/ @printf "\n run bandit::\n" && bandit -r --exclude .venv -ll . - @printf "\n run mypy::\n" && mypy --show-column-numbers -m naz.q -m naz.throttle -m naz.ratelimiter -m naz.hooks -m naz.sequence \ No newline at end of file + @printf "\n run mypy::\n" && mypy --show-column-numbers -m naz.q -m naz.throttle -m naz.ratelimiter -m naz.hooks -m naz.sequence + + +asciinema rec --idle-time-limit 2 first.cast +asciinema play -i 2 docs/first.cast +asciinema play -i 2 -s 2 docs/first.cast + + +--- +#### 1.2.1 codeee +![codeee](docs/2times.png) + + +--- +#### 1.2.2 codee2 +![codeee](docs/1times.png) + + +--- +![codeee](docs/2times.png) {this is what we will do} + + +--- +![codeee](docs/1times.png) + + +Use https://carbon.now.sh for code png's + + diff --git a/PITCHME.md b/PITCHME.md new file mode 100644 index 00000000..418f16db --- /dev/null +++ b/PITCHME.md @@ -0,0 +1,217 @@ +--- +## Introduction to naz, an async SMPP client + +https://gitpitch.com/komuw/naz/presentation + +--- +#### topics +1. SMPP +2. SMPP & python intro +3. current lay of the land +4. naz intro +5. naz features +6. Q&A + + +--- +#### 1. intro +**Name:** Komu Wairagu +**Occupation:** Software developer at [https://jumo.world/](https://jumo.world/) + +**About Me:** https://www.komu.engineer/about + +--- +#### 1.1 SMPP +Short Message Peer-to-Peer. +It's a protocol designed for transfer of Short messages between an SMS server and a mobile phone. +Based on exchange of request/response protocol data units(PDUs) between client & server over TCP/IP network. + + +--- +#### 1.2 Why care about SMPP? +Typically used for SMS and USSD by Telcos(Mobile Network Operators). +If you want to do integrate with various Telcos; you'll have to SMPP. + + +--- +#### 1.3 sequence of requests +![Image of sequence](docs/pyconKE2018/request-response-sequence.png) + + +--- +#### 1.4 PDU format +![Image of pdu format](docs/pyconKE2018/pdu-format.png) + + +--- +#### 2. SMPP & python +How do you connect to SMPP server(SMSC) from Python? + + +--- +![SMPP & python](docs/pyconKE2018/python-smpp-intro.png) + + +--- +#### 3. current lay of the land +- github.com/podshumok/python-smpplib +- github.com/praekelt/vumi +- ... couple more + +--- +#### 3.1 problems with current solutions + - complexity of code base + - coupling with other things(rabbitMQ, redis, Twisted) + - non-granular configurability + - (you can only set `throttle_delay: X seconds` ) + - maintenance debt: + - (we had to disable vumi sentry integration; + vumi outdated raven dependancies) + - cant migrate your app to Python3(because vumi is py2) + - lack of visibility(what is happenning?) + - (you have to enrich vumi logs). + + +--- +#### 4. naz intro +naz is an async SMPP client. +It's easily configurable, BYO(throttlers, rateLimiters etc) + +```bash +pip install naz +``` + +--- +#### 4.1 architecture +| Your App | ---> | Queue | ---> | Naz | +what is the Queue?? inMem, rabbitmq, redis ...?? +Naz makes no imposition of what the Queue is. +BYO queue... + + +--- +![naz exampleusage](docs/pyconKE2018/naz-example-usage.png) + + +--- +#### 4.2.1 sequence of requests +![Image of sequence](docs/pyconKE2018/request-response-sequence.png) + + +--- +#### 5. naz features +running theme: configurability, observability, BYO ... nini nini + + +--- +#### 5.1.1 observability: logging +![naz-observability-logging](docs/pyconKE2018/naz-observability-logging.png) + +--- +#### 5.1.1 observability: logs +![naz-observability-logs](docs/pyconKE2018/naz-observability-logs.png) + + + +--- +#### 5.1.2 observability: hooks +An instance of a class that implements `naz.hooks.BaseHook`. It has two methods `request` and `response`. +create an instance implementation of `BaseHook`, plug it in, and u can do whatever u want inside `request`/`response` methods. + +--- +![naz-observability-hooks-example1](docs/pyconKE2018/naz-observability-hooks.png) + + +--- +#### 5.1.2 observability: hooks example 2 +![naz-observability-hooks-prometheus](docs/pyconKE2018/naz-observability-hooks-prometheus.png) + + +--- +#### 5.2 Rate limiting +An instance of a class that implements `naz.ratelimiter.BaseRateLimiter`. It has one method `limit`. +create an instance implementation of `BaseRateLimiter`, plug it in, and u can implement any rate limiting algo inside `limit` method. +`naz` ships with a simple token-bucket Ratelimiter, `SimpleRateLimiter` + + +--- +#### 5.2 Rate limiting: example +![naz-ratelimiting-simple](docs/pyconKE2018/naz-ratelimiting-simple.png) + + +--- +#### 5.2 Rate limiting - logs +![naz-ratelimiting-simple-logs](docs/pyconKE2018/naz-ratelimiting-simple-logs.png) + + +--- +#### 5.2 Rate limiting: example2 +![naz-ratelimiting-awesome](docs/pyconKE2018/naz-ratelimiting-awesome.png) + + + +--- +#### 5.4 Throttle handling +An instance of a class that implements `naz.throttle.BaseThrottleHandler`. +`naz` calls it to handle throttling events from Telco. +`naz` ships with a default, `SimpleThrottleHandler` + +--- +#### 5.4 Throttle handling; example +![naz-throttle-handling](docs/pyconKE2018/naz-throttle-handling.png) + + +--- +#### 5.5 Queuing +An instance of a class that implements `naz.q.BaseOutboundQueue`. It has two methods `enqueue` & `dequeue`. +what you put inside those two methods is upto you. +Your app queues messages, naz consumes from that queue and then sends those messages to SMSC/server. +`naz` ships with a `SimpleOutboundQueue` that queues in Memory. + +--- +#### 5.5 Queuing; example +![naz-redis-queue](docs/pyconKE2018/naz-redis-queue.png) + + + +--- +#### 5.5 Queuing; example (your app) +![naz-redis-queue-app](docs/pyconKE2018/naz-redis-queue-app.png) + +--- +#### 5.6 cli app +this is installed when you install `naz`. +```sh +naz-cli --help +``` +```bash +usage: naz [-h] [--version] --config CONFIG + +naz is an SMPP client. +example usage: naz-cli --config /path/to/my_config.json + +optional arguments: + -h, --help show this help message and exit + --version The currently installed naz version. + --config CONFIG The config file to use. eg: --config + /path/to/my_config.json +``` + +--- +#### 5.6 cli app; example +demo + + +--- +#### 6. resources +- https://github.com/komuw/naz +- https://github.com/komuw/naz/blob/master/docs/SMPP_v3_4_specification.pdf +- https://gitpitch.com/komuw/naz/presentation +- https://github.com/praekelt/vumi + +- https://www.komu.engineer/about + + +--- +#### 6. Thanks +Q & A diff --git a/README.md b/README.md index a82aedec..4a69e8a9 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ naz is in active development and it's API may change in backward incompatible wa + [Throttle handling](#4-throttle-handling) + [Queuing](#5-queuing) +https://gitpitch.com/komuw/naz/presentation ## Installation diff --git a/docs/pyconKE2018/naz-example-usage.png b/docs/pyconKE2018/naz-example-usage.png new file mode 100644 index 00000000..52bd823c Binary files /dev/null and b/docs/pyconKE2018/naz-example-usage.png differ diff --git a/docs/pyconKE2018/naz-example-usage.py b/docs/pyconKE2018/naz-example-usage.py new file mode 100644 index 00000000..819d7e2b --- /dev/null +++ b/docs/pyconKE2018/naz-example-usage.py @@ -0,0 +1,26 @@ +import naz, asyncio + +loop = asyncio.get_event_loop() +outboundqueue = naz.q.SimpleOutboundQueue(maxsize=1000, loop=loop) +cli = naz.Client( + async_loop=loop, + smsc_host="127.0.0.1", + smsc_port=2775, + system_id="smppclient1", + password="password", + outboundqueue=outboundqueue, +) + +# 1. network connect and bind +reader, writer = loop.run_until_complete(cli.connect()) +loop.run_until_complete(cli.tranceiver_bind()) +try: + # 2. send SMS, read responses from SMSC, send status checks + tasks = asyncio.gather(cli.send_forever(), cli.receive_data(), cli.enquire_link()) + loop.run_until_complete(tasks) +except Exception as e: + print("exception occured. error={0}".format(str(e))) +finally: + # 3. unbind + loop.run_until_complete(cli.unbind()) + loop.close() diff --git a/docs/pyconKE2018/naz-observability-hooks-prometheus.png b/docs/pyconKE2018/naz-observability-hooks-prometheus.png new file mode 100644 index 00000000..f1a63fc6 Binary files /dev/null and b/docs/pyconKE2018/naz-observability-hooks-prometheus.png differ diff --git a/docs/pyconKE2018/naz-observability-hooks.png b/docs/pyconKE2018/naz-observability-hooks.png new file mode 100644 index 00000000..4762a0c7 Binary files /dev/null and b/docs/pyconKE2018/naz-observability-hooks.png differ diff --git a/docs/pyconKE2018/naz-observability-hooks.py b/docs/pyconKE2018/naz-observability-hooks.py new file mode 100644 index 00000000..85ac72a2 --- /dev/null +++ b/docs/pyconKE2018/naz-observability-hooks.py @@ -0,0 +1,40 @@ +import sqlite3 +import naz + +class SetMessageStateHook(naz.hooks.BaseHook): + async def request(self, smpp_event, correlation_id): + pass + async def response(self, smpp_event, correlation_id): + if smpp_event == "deliver_sm": + conn = sqlite3.connect('mySmsDB.db') + c = conn.cursor() + t = (correlation_id,) + c.execute("UPDATE \ + SmsTable \ + SET State='delivered' \ + WHERE CorrelatinID=?", t) + conn.commit() + conn.close() + +stateHook = SetMessageStateHook() +cli = naz.Client( + ... + hook=stateHook, +) + +import naz +from prometheus_client import Counter + +class MyPrometheusHook(naz.hooks.BaseHook): + async def request(self, smpp_event, correlation_id): + c = Counter('my_requests', 'Description of counter') + c.inc() # Increment by 1 + async def response(self, smpp_event, correlation_id): + c = Counter('my_responses', 'Description of counter') + c.inc() # Increment by 1 + +myHook = MyPrometheusHook() +cli = naz.Client( + ... + hook=myHook, +) \ No newline at end of file diff --git a/docs/pyconKE2018/naz-observability-logging.png b/docs/pyconKE2018/naz-observability-logging.png new file mode 100644 index 00000000..f140c343 Binary files /dev/null and b/docs/pyconKE2018/naz-observability-logging.png differ diff --git a/docs/pyconKE2018/naz-observability-logging.py b/docs/pyconKE2018/naz-observability-logging.py new file mode 100644 index 00000000..19288487 --- /dev/null +++ b/docs/pyconKE2018/naz-observability-logging.py @@ -0,0 +1,7 @@ +import naz +cli = naz.Client( + ... + log_metadata={ + "env": "prod", "release": "canary", "work": "jira-2345" + } +) \ No newline at end of file diff --git a/docs/pyconKE2018/naz-observability-logs.png b/docs/pyconKE2018/naz-observability-logs.png new file mode 100644 index 00000000..f1150e5c Binary files /dev/null and b/docs/pyconKE2018/naz-observability-logs.png differ diff --git a/docs/pyconKE2018/naz-ratelimiting-awesome.png b/docs/pyconKE2018/naz-ratelimiting-awesome.png new file mode 100644 index 00000000..1b8225fd Binary files /dev/null and b/docs/pyconKE2018/naz-ratelimiting-awesome.png differ diff --git a/docs/pyconKE2018/naz-ratelimiting-simple-logs.png b/docs/pyconKE2018/naz-ratelimiting-simple-logs.png new file mode 100644 index 00000000..f6794898 Binary files /dev/null and b/docs/pyconKE2018/naz-ratelimiting-simple-logs.png differ diff --git a/docs/pyconKE2018/naz-ratelimiting-simple.png b/docs/pyconKE2018/naz-ratelimiting-simple.png new file mode 100644 index 00000000..414ef358 Binary files /dev/null and b/docs/pyconKE2018/naz-ratelimiting-simple.png differ diff --git a/docs/pyconKE2018/naz-ratelimiting.py b/docs/pyconKE2018/naz-ratelimiting.py new file mode 100644 index 00000000..8ce34ef9 --- /dev/null +++ b/docs/pyconKE2018/naz-ratelimiting.py @@ -0,0 +1,22 @@ +import naz +limiter = naz.ratelimiter.SimpleRateLimiter( + send_rate=1, max_tokens=1, delay_for_tokens=6 +) +cli = naz.Client( + ... + rateLimiter=limiter, +) + + +import naz +class AwesomeLimiter(naz.ratelimiter.BaseRateLimiter): + async def limit(self): + sleeper = 13.13 + print("\n\t rate limiting. sleep={}".format(sleeper)) + await asyncio.sleep(sleeper) + +lim = AwesomeLimiter() +cli = naz.Client( + ... + rateLimiter=lim, +) diff --git a/docs/pyconKE2018/naz-redis-queue-app.png b/docs/pyconKE2018/naz-redis-queue-app.png new file mode 100644 index 00000000..b41a96cf Binary files /dev/null and b/docs/pyconKE2018/naz-redis-queue-app.png differ diff --git a/docs/pyconKE2018/naz-redis-queue.png b/docs/pyconKE2018/naz-redis-queue.png new file mode 100644 index 00000000..5068497b Binary files /dev/null and b/docs/pyconKE2018/naz-redis-queue.png differ diff --git a/docs/pyconKE2018/naz-redis-queue.py b/docs/pyconKE2018/naz-redis-queue.py new file mode 100644 index 00000000..eec3e81b --- /dev/null +++ b/docs/pyconKE2018/naz-redis-queue.py @@ -0,0 +1,38 @@ +mport asyncio, naz, redis + +class RedisExampleQueue(naz.q.BaseOutboundQueue): + def __init__(self): + self.redis_instance = redis.StrictRedis(host="localhost", port=6379, db=0) + self.queue_name = "myqueue" + async def enqueue(self, item): + self.redis_instance.lpush(self.queue_name, json.dumps(item)) + async def dequeue(self): + val = self.redis_instance.brpop(self.queue_name) + dequed_item = json.loads(val[1].decode()) + return dequed_item + +myQueue = RedisExampleQueue() + +loop = asyncio.get_event_loop() +cli = naz.Client( + async_loop=loop, + smsc_host="127.0.0.1", + smsc_port=2775, + system_id="smppclient1", + password="password", + outboundqueue=myQueue, +) + +# in your app +import asyncio + +myQueue = RedisExampleQueue() +message_data = { + "smpp_event": "submit_sm", + "short_message": "Hello, Thank you for subscribing to our Service.", + "correlation_id": "myid12345", + "source_addr": "254722111111", + "destination_addr": "254722999999", +} +loop = asyncio.get_event_loop() +loop.run_until_complete(myQueue.enqueue(message_data)) \ No newline at end of file diff --git a/docs/pyconKE2018/naz-throttle-handling.png b/docs/pyconKE2018/naz-throttle-handling.png new file mode 100644 index 00000000..7f6d414d Binary files /dev/null and b/docs/pyconKE2018/naz-throttle-handling.png differ diff --git a/docs/pyconKE2018/naz-throttle-handling.py b/docs/pyconKE2018/naz-throttle-handling.py new file mode 100644 index 00000000..0ef63f2a --- /dev/null +++ b/docs/pyconKE2018/naz-throttle-handling.py @@ -0,0 +1,9 @@ +import naz + +TH = naz.throttle.SimpleThrottleHandler(sampling_period=180, + sample_size=45, + deny_request_at=1.2) +cli = naz.Client( + ... + throttle_handler=TH, +) \ No newline at end of file diff --git a/docs/pyconKE2018/pdu-format.png b/docs/pyconKE2018/pdu-format.png new file mode 100644 index 00000000..d7b856db Binary files /dev/null and b/docs/pyconKE2018/pdu-format.png differ diff --git a/docs/pyconKE2018/pdu-format.txt b/docs/pyconKE2018/pdu-format.txt new file mode 100644 index 00000000..6b0f0423 --- /dev/null +++ b/docs/pyconKE2018/pdu-format.txt @@ -0,0 +1,8 @@ + +< ----------------------------- HEADER -------------------------> <---- BODY ---> +| command_length | command_id | command_status | sequence_number | BODY | + +1. command_length, 4bytes, Integer +2. command_id, 4bytes, Integer. eg `submit_sm` is Integer 4 +3. command_status, 4bytes, Integer. eg success is Integer 0 +4. sequence_number, 4bytes, Integer \ No newline at end of file diff --git a/docs/pyconKE2018/pdu_format.png b/docs/pyconKE2018/pdu_format.png new file mode 100644 index 00000000..328935d3 Binary files /dev/null and b/docs/pyconKE2018/pdu_format.png differ diff --git a/docs/pyconKE2018/pdu_format.txt b/docs/pyconKE2018/pdu_format.txt new file mode 100644 index 00000000..fcbdabc5 --- /dev/null +++ b/docs/pyconKE2018/pdu_format.txt @@ -0,0 +1,8 @@ + +<---------------------- HEADER -------------------------------> +| command_length | command_id | command_status | sequence_number | BODY | + +1. command_length, 4bytes, Integer +2. command_id, 4bytes, Integer. eg `submit_sm` is command_id 4 +3. command_status, 4bytes, Integer. eg success is integer 0 +4. sequence_number, 4bytes, Integer diff --git a/docs/pyconKE2018/python-smpp-intro.png b/docs/pyconKE2018/python-smpp-intro.png new file mode 100644 index 00000000..9cdf0e75 Binary files /dev/null and b/docs/pyconKE2018/python-smpp-intro.png differ diff --git a/docs/pyconKE2018/python-smpp-intro.py b/docs/pyconKE2018/python-smpp-intro.py new file mode 100644 index 00000000..c6d7834d --- /dev/null +++ b/docs/pyconKE2018/python-smpp-intro.py @@ -0,0 +1,18 @@ +# 1. connect to network +import socket, struct + +sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +sock.settimeout(5) +sock.connect(("127.0.0.1", 2775)) + +# 2. create PDU +body = b"" +command_length = 16 + len(body) # 16 is for headers +command_id = 21 # enquire_link PDU +command_status = 0 +sequence_number = 1 +header = struct.pack(">IIII", command_length, command_id, command_status, sequence_number) +full_pdu = header + body + +# 3. send PDU over network +sock.send(full_pdu) diff --git a/docs/pyconKE2018/request-response-sequence.png b/docs/pyconKE2018/request-response-sequence.png new file mode 100644 index 00000000..3745741c Binary files /dev/null and b/docs/pyconKE2018/request-response-sequence.png differ diff --git a/docs/pyconKE2018/request-response-sequence.txt b/docs/pyconKE2018/request-response-sequence.txt new file mode 100644 index 00000000..a6333d1b --- /dev/null +++ b/docs/pyconKE2018/request-response-sequence.txt @@ -0,0 +1,20 @@ + +| Your App | | Telco SMSC server | | Mobile Phone | + + 1. network connect +----------------------------------------> + 2. bind_transceiver +----------------------------------------> + 3. bind_transceiver_resp +<---------------------------------------- + 4. submit_sm +----------------------------------------> Send SMS to Phone + ----------------------------------------> + 5. submit_sm_resp +<---------------------------------------- +.... +.... + 34. unbind +----------------------------------------> + 35. unbind_resp +<---------------------------------------- \ No newline at end of file diff --git a/docs/pyconKE2018/sequence.png b/docs/pyconKE2018/sequence.png new file mode 100644 index 00000000..65d219f9 Binary files /dev/null and b/docs/pyconKE2018/sequence.png differ diff --git a/docs/pyconKE2018/sequence.txt b/docs/pyconKE2018/sequence.txt new file mode 100644 index 00000000..820afa99 --- /dev/null +++ b/docs/pyconKE2018/sequence.txt @@ -0,0 +1,28 @@ +| CLIENT | | SERVER/smsc | | PHONE | + 1. network connect + -----------------------------> + + 2. bind_transceiver + -----------------------------> + 3. bind_transceiver_resp + <----------------------------- + + 4. submit_sm + -----------------------------> + 5. submit_sm_resp send SMS to phone + <----------------------------- ----------------------> + + 6. submit_sm + -----------------------------> + 7. submit_sm + -----------------------------> + 8. submit_sm + -----------------------------> + 9. submit_sm_resp + <----------------------------- + + 10. unbind + -----------------------------> + 11. unbind_resp + <----------------------------- + diff --git a/examples/example_config.json b/examples/example_config.json index 5e9e7c5a..c592e38b 100644 --- a/examples/example_config.json +++ b/examples/example_config.json @@ -5,13 +5,13 @@ "password": "password", "outboundqueue": "examples.example_klasses.ExampleQueueInstance", "encoding": "gsm0338", - "sequence_generator": "examples.example_klasses.ExampleSeqGen", "loglevel": "INFO", "log_metadata": { "environment": "production", - "release": "canary" + "release": "canary", + "age": 67 }, "codec_errors_level": "ignore", - "enquire_link_interval": 30, - "rateLimiter": "examples.example_klasses.ExampleRateLimiter" + "enquire_link_interval": 70, + "rateLimiter": "examples.example_klasses.MyRateLimiter" } \ No newline at end of file diff --git a/examples/in_mem_queue_example.py b/examples/in_mem_queue_example.py index df752958..ec4fa913 100644 --- a/examples/in_mem_queue_example.py +++ b/examples/in_mem_queue_example.py @@ -2,9 +2,16 @@ import naz +import logging + +logger = logging.getLogger() + loop = asyncio.get_event_loop() outboundqueue = naz.q.SimpleOutboundQueue(maxsize=1000, loop=loop) +limiter = naz.ratelimiter.SimpleRateLimiter( + logger=logger, send_rate=1, max_tokens=1, delay_for_tokens=6 +) cli = naz.Client( async_loop=loop, smsc_host="127.0.0.1", @@ -12,10 +19,11 @@ system_id="smppclient1", password="password", outboundqueue=outboundqueue, + rateLimiter=limiter, ) # queue messages to send -for i in range(0, 4): +for i in range(0, 12): print("submit_sm round:", i) item_to_enqueue = { "smpp_event": "submit_sm",