Skip to content

Commit bc66a8f

Browse files
authored
Merge pull request #4700 from telefonicaid/feature/4666-kafka-subscriptions
Feature/4666 kafka subscriptions
2 parents ccbf05d + bdd41c4 commit bc66a8f

File tree

105 files changed

+6532
-273
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

105 files changed

+6532
-273
lines changed

.github/workflows/codeql.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ jobs:
9090
export ORION_SRC=$(pwd)
9191
# Install dependencies
9292
sudo apt-get update
93-
sudo apt-get install make cmake g++ libssl-dev libcurl4-openssl-dev libboost-dev libboost-regex-dev libboost-filesystem-dev libboost-thread-dev uuid-dev libgnutls28-dev libsasl2-dev libgcrypt-dev
93+
sudo apt-get install make cmake g++ libssl-dev libcurl4-openssl-dev libboost-dev libboost-regex-dev libboost-filesystem-dev libboost-thread-dev uuid-dev libgnutls28-dev libsasl2-dev libgcrypt-dev librdkafka-dev zlib1g-dev
9494
# Install libmicrohttpd from source
9595
cd /opt
9696
curl -kOL https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz

.github/workflows/functional.yml

Lines changed: 28 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -9,54 +9,49 @@ on:
99
- master
1010
- '!checkvalgrind**'
1111

12-
concurrency:
12+
concurrency:
1313
group: ${{ github.workflow }}-${{ github.ref }}
1414
cancel-in-progress: true
1515

1616
env:
17-
TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
17+
TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
1818

1919
jobs:
2020
functional:
2121
runs-on: ubuntu-22.04
2222
continue-on-error: true
2323

24-
services:
25-
mongodb:
26-
image: mongo:8.0
27-
ports:
28-
- 27017:27017
29-
30-
name: functional - ${{ matrix.payload.name }}
31-
3224
strategy:
3325
matrix:
34-
payload:
35-
- { name: 'batch 1 - cache', range: '-e FT_FROM_IX=0 -e FT_TO_IX=300' }
36-
- { name: 'batch 2 - cache', range: '-e FT_FROM_IX=301 -e FT_TO_IX=600' }
37-
- { name: 'batch 3 - cache', range: '-e FT_FROM_IX=601 -e FT_TO_IX=900' }
38-
- { name: 'batch 4 - cache', range: '-e FT_FROM_IX=901 -e FT_TO_IX=1200' }
39-
- { name: 'batch 5 - cache', range: '-e FT_FROM_IX=1201' }
40-
- { name: 'batch 1 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=0 -e FT_TO_IX=300' }
41-
- { name: 'batch 2 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=301 -e FT_TO_IX=600' }
42-
- { name: 'batch 3 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=601 -e FT_TO_IX=900' }
43-
- { name: 'batch 4 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=901 -e FT_TO_IX=1200' }
44-
- { name: 'batch 5 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=1201' }
26+
batch:
27+
- { name: "batch 1 - cache", FT_FROM_IX: 0, FT_TO_IX: 300, CB_NO_CACHE: "" }
28+
- { name: "batch 2 - cache", FT_FROM_IX: 301, FT_TO_IX: 600, CB_NO_CACHE: "" }
29+
- { name: "batch 3 - cache", FT_FROM_IX: 601, FT_TO_IX: 900, CB_NO_CACHE: "" }
30+
- { name: "batch 4 - cache", FT_FROM_IX: 901, FT_TO_IX: 1200, CB_NO_CACHE: "" }
31+
- { name: "batch 5 - cache", FT_FROM_IX: 1201,FT_TO_IX: "", CB_NO_CACHE: "" }
32+
- { name: "batch 1 - no cache", FT_FROM_IX: 0, FT_TO_IX: 300, CB_NO_CACHE: "ON" }
33+
- { name: "batch 2 - no cache", FT_FROM_IX: 301, FT_TO_IX: 600, CB_NO_CACHE: "ON" }
34+
- { name: "batch 3 - no cache", FT_FROM_IX: 601, FT_TO_IX: 900, CB_NO_CACHE: "ON" }
35+
- { name: "batch 4 - no cache", FT_FROM_IX: 901, FT_TO_IX: 1200, CB_NO_CACHE: "ON" }
36+
- { name: "batch 5 - no cache", FT_FROM_IX: 1201,FT_TO_IX: "", CB_NO_CACHE: "ON" }
37+
38+
name: functional - ${{ matrix.batch.name }}
4539

4640
steps:
4741
- uses: actions/checkout@v2
4842

49-
- name: Run Mosquitto container with authenticated user
50-
# Why don't to use a service for this (as moquitto-extra above)? It would be the desirable solution, but I haven't able to
51-
# configure volumes properly for it (the mosquitto container as service has problems to get the configuration files). Feedback welcome! :)
52-
run: docker run -d --name mosquitto -p 1883:1883 -v $(pwd)/test/functionalTest/mosquittoConf:/mosquitto/config:ro eclipse-mosquitto:2.0.11
53-
54-
- name: Run Extra Mosquitto container used in multibroker tests
55-
# Note we are reusing the same volume (although the extra broker does not require authenticated user). We use a service in the past
56-
# to implement this (using eclipse-mosquitto:1.6.15) but in eclipse-mosquitto:2.0.11 it seems something in the internal configuration
57-
# has changed and it doesn't work if we don't provide the configuration files in the volume. Again, feedback is welcome! :)
58-
run: docker run -d --name mosquitto-extra -p 1884:1883 -v $(pwd)/test/functionalTest/mosquittoConf:/mosquitto/config:ro eclipse-mosquitto:2.0.11
59-
6043
- name: Run functional test
44+
env:
45+
REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
46+
FT_FROM_IX: ${{ matrix.batch.FT_FROM_IX }}
47+
FT_TO_IX: ${{ matrix.batch.FT_TO_IX }}
48+
CB_NO_CACHE: ${{ matrix.batch.CB_NO_CACHE }}
49+
TEST_IMAGE_NAME: ${{ env.TEST_IMAGE_NAME }}
6150
run: |
62-
docker run --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts functional
51+
docker compose \
52+
--project-name functional-${{ github.run_id }}-${{ strategy.job-index }} \
53+
-f ./ci/deb/docker-compose-ci.yml -f ./ci/deb/docker-compose-ci.functional.yml \
54+
--project-directory . \
55+
up --build --abort-on-container-exit --exit-code-from tests \
56+
--attach tests \
57+
--no-log-prefix

.github/workflows/valgrind-nocache.yml

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -24,37 +24,32 @@ jobs:
2424
runs-on: ubuntu-22.04
2525
continue-on-error: true
2626

27-
services:
28-
mongodb:
29-
image: mongo:8.0
30-
ports:
31-
- 27017:27017
32-
33-
name: valgrind - ${{ matrix.payload.name }}
34-
3527
strategy:
3628
matrix:
37-
payload:
38-
- { name: 'batch 1 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=0 -e FT_TO_IX=300'}
39-
- { name: 'batch 2 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=301 -e FT_TO_IX=600'}
40-
- { name: 'batch 3 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=601 -e FT_TO_IX=900'}
41-
- { name: 'batch 4 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=901 -e FT_TO_IX=1200'}
42-
- { name: 'batch 5 - no cache', range: '-e CB_NO_CACHE=ON -e FT_FROM_IX=1201'}
29+
batch:
30+
- { name: "batch 1 - no cache", FT_FROM_IX: 0, FT_TO_IX: 300, CB_NO_CACHE: "ON" }
31+
- { name: "batch 2 - no cache", FT_FROM_IX: 301, FT_TO_IX: 600, CB_NO_CACHE: "ON" }
32+
- { name: "batch 3 - no cache", FT_FROM_IX: 601, FT_TO_IX: 900, CB_NO_CACHE: "ON" }
33+
- { name: "batch 4 - no cache", FT_FROM_IX: 901, FT_TO_IX: 1200, CB_NO_CACHE: "ON" }
34+
- { name: "batch 5 - no cache", FT_FROM_IX: 1201,FT_TO_IX: "", CB_NO_CACHE: "ON" }
4335

36+
name: valgrind - ${{ matrix.batch.name }}
37+
4438
steps:
4539
- uses: actions/checkout@v2
4640

47-
- name: Run Mosquitto container with authenticated user
48-
# Why don't to use a service for this (as moquitto-extra above)? It would be the desirable solution, but I haven't able to
49-
# configure volumes properly for it (the mosquitto container as service has problems to get the configuration files). Feedback welcome! :)
50-
run: docker run -d --name mosquitto -p 1883:1883 -v $(pwd)/test/functionalTest/mosquittoConf:/mosquitto/config:ro eclipse-mosquitto:2.0.11
51-
52-
- name: Run Extra Mosquitto container used in multibroker tests
53-
# Note we are reusing the same volume (although the extra broker does not require authenticated user). We use a service in the past
54-
# to implement this (using eclipse-mosquitto:1.6.15) but in eclipse-mosquitto:2.0.11 it seems something in the internal configuration
55-
# has changed and it doesn't work if we don't provide the configuration files in the volume. Again, feedback is welcome! :)
56-
run: docker run -d --name mosquitto-extra -p 1884:1883 -v $(pwd)/test/functionalTest/mosquittoConf:/mosquitto/config:ro eclipse-mosquitto:2.0.11
57-
5841
- name: Run valgrind test
42+
env:
43+
REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
44+
FT_FROM_IX: ${{ matrix.batch.FT_FROM_IX }}
45+
FT_TO_IX: ${{ matrix.batch.FT_TO_IX }}
46+
CB_NO_CACHE: ${{ matrix.batch.CB_NO_CACHE }}
47+
TEST_IMAGE_NAME: ${{ env.TEST_IMAGE_NAME }}
5948
run: |
60-
docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind
49+
docker compose \
50+
--project-name valgrindnocache-${{ github.run_id }}-${{ strategy.job-index }} \
51+
-f ./ci/deb/docker-compose-ci.yml -f ./ci/deb/docker-compose-ci.valgrind.yml \
52+
--project-directory . \
53+
up --build --abort-on-container-exit --exit-code-from tests \
54+
--attach tests \
55+
--no-log-prefix

.github/workflows/valgrind.yml

Lines changed: 22 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -21,37 +21,32 @@ jobs:
2121
runs-on: ubuntu-22.04
2222
continue-on-error: true
2323

24-
services:
25-
mongodb:
26-
image: mongo:8.0
27-
ports:
28-
- 27017:27017
29-
30-
name: valgrind - ${{ matrix.payload.name }}
31-
3224
strategy:
3325
matrix:
34-
payload:
35-
- { name: 'batch 1 - cache', range: '-e FT_FROM_IX=0 -e FT_TO_IX=300'}
36-
- { name: 'batch 2 - cache', range: '-e FT_FROM_IX=301 -e FT_TO_IX=600'}
37-
- { name: 'batch 3 - cache', range: '-e FT_FROM_IX=601 -e FT_TO_IX=900'}
38-
- { name: 'batch 4 - cache', range: '-e FT_FROM_IX=901 -e FT_TO_IX=1200'}
39-
- { name: 'batch 5 - cache', range: '-e FT_FROM_IX=1201'}
40-
41-
steps:
42-
- uses: actions/checkout@v2
26+
batch:
27+
- { name: "batch 1 - cache", FT_FROM_IX: 0, FT_TO_IX: 300, CB_NO_CACHE: "" }
28+
- { name: "batch 2 - cache", FT_FROM_IX: 301, FT_TO_IX: 600, CB_NO_CACHE: "" }
29+
- { name: "batch 3 - cache", FT_FROM_IX: 601, FT_TO_IX: 900, CB_NO_CACHE: "" }
30+
- { name: "batch 4 - cache", FT_FROM_IX: 901, FT_TO_IX: 1200, CB_NO_CACHE: "" }
31+
- { name: "batch 5 - cache", FT_FROM_IX: 1201,FT_TO_IX: "", CB_NO_CACHE: "" }
4332

44-
- name: Run Mosquitto container with authenticated user
45-
# Why don't to use a service for this (as moquitto-extra above)? It would be the desirable solution, but I haven't able to
46-
# configure volumes properly for it (the mosquitto container as service has problems to get the configuration files). Feedback welcome! :)
47-
run: docker run -d --name mosquitto -p 1883:1883 -v $(pwd)/test/functionalTest/mosquittoConf:/mosquitto/config:ro eclipse-mosquitto:2.0.11
33+
name: valgrind - ${{ matrix.batch.name }}
4834

49-
- name: Run Extra Mosquitto container used in multibroker tests
50-
# Note we are reusing the same volume (although the extra broker does not require authenticated user). We use a service in the past
51-
# to implement this (using eclipse-mosquitto:1.6.15) but in eclipse-mosquitto:2.0.11 it seems something in the internal configuration
52-
# has changed and it doesn't work if we don't provide the configuration files in the volume. Again, feedback is welcome! :)
53-
run: docker run -d --name mosquitto-extra -p 1884:1883 -v $(pwd)/test/functionalTest/mosquittoConf:/mosquitto/config:ro eclipse-mosquitto:2.0.11
35+
steps:
36+
- uses: actions/checkout@v2
5437

5538
- name: Run valgrind test
39+
env:
40+
REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
41+
FT_FROM_IX: ${{ matrix.batch.FT_FROM_IX }}
42+
FT_TO_IX: ${{ matrix.batch.FT_TO_IX }}
43+
CB_NO_CACHE: ${{ matrix.batch.CB_NO_CACHE }}
44+
TEST_IMAGE_NAME: ${{ env.TEST_IMAGE_NAME }}
5645
run: |
57-
docker run --privileged --network host -t --rm -e REPO_ACCESS_TOKEN=${{ secrets.REPO_ACCESS_TOKEN }} ${{ matrix.payload.range }} -v $(pwd):/opt/fiware-orion ${{ env.TEST_IMAGE_NAME }} build -miqts valgrind
46+
docker compose \
47+
--project-name valgrind-${{ github.run_id }}-${{ strategy.job-index }} \
48+
-f ./ci/deb/docker-compose-ci.yml -f ./ci/deb/docker-compose-ci.valgrind.yml \
49+
--project-directory . \
50+
up --build --abort-on-container-exit --exit-code-from tests \
51+
--attach tests \
52+
--no-log-prefix

CHANGES_NEXT_RELEASE

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,5 @@
99
- Fix: csubs and registrations now use isPattern as bool instead of string (legacy from NGSIv1 model), although strings are still supported (#4681)
1010
- Fix: use warning instead of Runtime Error when adding an object or array in basic expressions mode
1111
- Hardening: remove a huge amount of NGSIv1 related dead-code (#4681)
12+
- Add: support for Kafka based notifications (#4666)
13+
- Add: new CLI parameters kafkaTimeout and kafkaMaxAge (and associated env vars ORION_KAFKA_TIMEOUT and ORION_KAFKA_MAX_AGE) (#4666)

CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ SET (ORION_LIBS
209209
lm
210210
pa
211211
mqtt
212+
kafka
212213
)
213214

214215
SET (BOOST
@@ -255,6 +256,7 @@ endif (HAVE_CJEXL)
255256

256257
SET (DYNAMIC_LIBS
257258
curl
259+
rdkafka
258260
gnutls
259261
pthread
260262
gcrypt
@@ -282,6 +284,7 @@ endif (UNIT_TEST)
282284
include_directories("/usr/include")
283285

284286
# Needed for the new C driver
287+
include_directories("/usr/include/librdkafka")
285288
include_directories("/usr/local/include/libmongoc-1.0")
286289
include_directories("/usr/local/include/libbson-1.0")
287290

@@ -333,6 +336,7 @@ if (error EQUAL 0)
333336
ADD_SUBDIRECTORY(src/lib/expressions)
334337
ADD_SUBDIRECTORY(src/lib/logSummary)
335338
ADD_SUBDIRECTORY(src/lib/mqtt)
339+
ADD_SUBDIRECTORY(src/lib/kafka)
336340
ADD_SUBDIRECTORY(src/app/contextBroker)
337341
else ()
338342
MESSAGE("cmake: NOT OK")

ci/README.md

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,39 @@ Current version of CI supports:
2222

2323
File compliance, payload and style checks are combined in one 'compliance' test.
2424

25+
# How to run all test locally
26+
27+
You can run the functional tests with:
28+
29+
```
30+
docker compose -f ci/deb/docker-compose-ci.yml -f ci/deb/docker-compose-ci.functional.yml --project-directory . up
31+
```
32+
33+
That would set up all the needed services (MongoDB, MQTT broker, Kafka broker, etc.) with the `docker-compose-ci.yml` and run the tests with `docker-compose-ci.functional.yml` (which uses the `telefonicaiot/fiware-orion:ci` for that).
34+
35+
Alternativelly you can run the valgrind tests with:
36+
37+
```
38+
docker compose -f ci/deb/docker-compose-ci.yml -f ci/deb/docker-compose-ci.valgrind.yml --project-directory . up
39+
```
40+
41+
For alternative ways of using the image, read next section.
42+
2543
# How to use the image locally
2644

2745
Sometime you need to run the CI image locally (for instance, to debug problems found in GitHub Action jobs). In that case,
2846
the following cheatsheet can be useful:
2947

30-
To download the image:
48+
First, you have to set up required services running in a terminal (stop local instances of the service before or you will get conflicts)
49+
50+
```
51+
cd /path/to/fiware-orion
52+
docker compose -f ci/deb/docker-compose-ci.yml --project-directory . up
53+
```
54+
55+
When you end your testing you can stop the services in the terminal typically with Ctrl+C.
56+
57+
Next, to download the image (alternativelly you can [build it locally](#how-to-build-the-image-locally))
3158

3259
```
3360
docker pull telefonicaiot/fiware-orion:ci
@@ -36,23 +63,21 @@ docker pull telefonicaiot/fiware-orion:ci
3663
To run the image in the same way that GitHub Actions does, for instance:
3764

3865
```
39-
# Check that MongoDB server is running in your localhost:27017
4066
cd /path/to/fiware-orion
41-
docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1201 -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci build -miqts functional
67+
docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1001 -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci build -miqts functional
4268
```
4369

4470
To run the image using an interactive bash on it
4571

4672
```
47-
# Check that MongoDB server is running in your localhost:27017
4873
cd /path/to/fiware-orion
4974
docker run --network host -ti -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci bash
5075
```
5176

5277
Once have a bash shell, you can do the same execution:
5378

5479
```
55-
root@debian11:/opt# CB_NO_CACHE=ON FT_FROM_IX=1201 build -miqts functional
80+
root@debian11:/opt# CB_NO_CACHE=ON FT_FROM_IX=1001 build -miqts functional
5681
```
5782

5883
Alternatively, you can run the `testHarness.sh` script directly (for instance, to execute a single test):
@@ -72,3 +97,20 @@ sudo chown -R fermin:fermin /path/to/fiware-orion
7297

7398
**NOTE2:** the `build` script will make changes in your `makefile` file. Do a `git checkout makefile` after your debug session
7499
to recover it.
100+
101+
# How to build the image locally
102+
103+
In some case, we may want to build the image locally instead of pull the one at Dockerhub. The following commands could help.
104+
105+
First, it's a good idea to remove any existing `telefonicaiot/fiware-orion:ci` image downloaed from Dockerhub:
106+
107+
```
108+
docker rmi <id of the image>
109+
```
110+
111+
Next, to build the container:
112+
113+
```
114+
cd /path/to/fiware-orion/ci/deb
115+
docker build -t telefonicaiot/fiware-orion:ci .
116+
```

ci/deb/Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ ADD fuse_gtest_files.py.patch /opt/archive/
88
RUN ln -s /opt/bin/build.sh /usr/local/bin/build \
99
&& /opt/bin/build-dep.sh
1010

11+
ENV PATH="/opt/kafka/bin:$PATH"
12+
1113
WORKDIR /opt/
1214

1315
CMD ["/usr/local/bin/build", "-h"]

ci/deb/build-dep.sh

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,12 @@ apt-get -y install \
4747
uuid-dev \
4848
libgnutls28-dev \
4949
libsasl2-dev \
50-
libgcrypt-dev
50+
libgcrypt-dev \
51+
librdkafka-dev \
52+
openjdk-17-jre-headless \
53+
zlib1g-dev\
54+
libzstd-dev \
55+
liblz4-dev
5156

5257
echo "INSTALL: MongoDB shell" \
5358
&& curl -fsSL https://www.mongodb.org/static/pgp/server-8.0.asc | gpg -o /usr/share/keyrings/mongodb-server-8.0.gpg --dearmor \
@@ -63,6 +68,7 @@ echo "INSTALL: python special dependencies" \
6368
&& pip install Werkzeug==2.0.2 \
6469
&& pip install paho-mqtt==1.6.1 \
6570
&& pip install amqtt==0.11.0b1 \
71+
&& pip install confluent-kafka==2.11.0 \
6672
&& deactivate
6773

6874
# Recommended setting for DENABLE_AUTOMATIC_INIT_AND_CLEANUP, to be removed in 2.0.0
@@ -104,6 +110,12 @@ echo "INSTALL: mosquitto" \
104110
&& make \
105111
&& make install
106112

113+
# Note in this case the directory created in /opt contains the software itself
114+
# (i.e. there isn't a install step itself). Note also due to this there isn't a removal (rm) at the end
115+
echo "INSTALL: Kafka" \
116+
&& curl -fsSL "https://downloads.apache.org/kafka/3.9.1/kafka_2.12-3.9.1.tgz" | tar xzC /opt \
117+
&& mv /opt/kafka_2.12-3.9.1 /opt/kafka
118+
107119
ldconfig
108120

109121
apt-get -y clean \

0 commit comments

Comments
 (0)