diff --git a/src/01_Introduction_Big_Data.md b/src/01_Introduction_Big_Data.md index fd7cfd2..2a27c80 100644 --- a/src/01_Introduction_Big_Data.md +++ b/src/01_Introduction_Big_Data.md @@ -1,7 +1,7 @@ --- title: Introduction to Big Data and its Ecosystem author: Guillaume Eynard-Bontemps, CNES (Centre National d'Etudes Spatiales - French Space Agency) -date: 2025 +date: 2026 --- # What is Big Data? @@ -56,9 +56,11 @@ Volume, variety, multiple sources, internal, external... ### Tools and technology -Store, Compute, Analyse: Calculators, Cloud, Hadoop, Spark, Dask +Store: Cloud storage, Hadoop, Specialized Hardware -Visualize, Use: Applications, Web interfaces +Compute, Analyse: Calculators, Cloud, Hadoop, Spark, Dask, Jupyter + +Visualize, Use: Applications, Web interfaces, Dashboards ::: ::: {.column width="50%"} @@ -82,9 +84,9 @@ What is the estimated size of the global data sphere? - Answer B: 175 Exabytes - Answer C: 175 Zetabytes -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAG4UlEQVR42u3d2Y7bOBBAUWmi///jgfLQQMPxoq1cLNI+5ynIxG2Z1rXoHoKa13WdgDrLNE3zPH/eCwt+uGyPSc+fXJF3M3XQgk/9kWfpzwv/z+cQ1BIhiBBECIgQRAiIEEQIiBC+z7L7L7pdHRJcQpG6AqNw2Urk4cEx2X7q1AEf+ix1JQTTURAhIEIQISBCECEgQhAh0NgSfPy4yyAiyztSV5YEBzzyuoJLXrrdBqbzs9SVEExHQYSACEGEgAhBhIAIQYRAY8vXvvJBbyGUuoooeC+qcW9l5UoIIgRECCIERAgiBEQIIgRECCIE2vneZWvd7rY07qvGlRBECIgQRAiIEEQIiBBECIgQhhBdMTPu7j2FuxIVrqeJvOpx3+vOj9yVEEQIIgRECCIERAgiBEQIIgQa218xY2eRt4ssWync3ib1qYOLWoY+S10JQYQgQkCEIEJAhCBCQIQgQqCxedyNQwYe9MDyjt33q3DvnMKndiUERAgiBEQIIgRECCIERAgiBEQIw1imjjfJKbyFWOqRd/uqg+vOUo88bxep8g2sXAnBdBRECIgQRAiIEEQIiBBECDQ2BzcOKtT5MoiydzRzF6luT5VuF+sc+eGuhGA6CiIERAgiBEQIIgRECCIEGpvXdU3dn2Nb3g9PXQlU+8PLzpXYIqTUZ488de3rciUE01EQoSEAEYIIARGCCAERgggBEcLX2V+2FpG6cVDhIqnCYSncwKrbMak9Nhs9gekoIEIQISBCECEgQhAhIEIYTu6KmV15a0d63uhp0K2cxr0hXOcrgVwJwXQURAiIEEQIiBBECIgQRAg0NqeuLElVuIFN6kYvhYPW7Xvd84kUzMceM2A6CiI0BCBCECEgQhAhIEIQIdDeMmXuthJcWbItdXnHuHf5qTrsbIOu5rHHDJiOAiIEEQIiBBECIgQRAiIEEQL3ijd6ylsJlbpiLvvY8g479eZnTpVrP9yVEExHQYSACEGEgAhBhIAIQYRAY3PtfaG2FW6IVHhrtEFvqzYlL8epjCT5LnquhGA6CiIERAgiBEQIIgRECCIEGpvXdc1bhFG4xiJ1UUv8hXd6NrghXHNWzIDpKIjQEIAIQYSACEGEgAhBhEB7y+6/SF22EtHz8o5u77uUNyY9v1+uhIAIQYSACEGEgAhBhIAIQYSACKEjS/DxhRsiFa4sCypcCVi4d1beY6f8u5clPbUrIZiOAiIEEYIIARGCCAERggiBEvu3Rut2o6eg1K2B8oa0cMx73uhp6BPJlRBMR0GEgAhBhIAIQYSACEGEQGPLlLmAo+cbVnW7K8kHj3nVYQcXMEXeryOniishmI6CCAERgggBEYIIARGCCAERwndZpg/dgafn1Vupi9pSl+MVKtyOLPuWb66EYDoKIgRECCIERAgiBEQIIgQaW3b/RbdLT4LLOwpfV+TIU+9P1vPNzz54OzJXQjAdBRECIgQRAiIEEQIiBBECjS3Bx497l69u91Mp3AYmuJ4mcuTjbm8TP4ddCcF0FEQIiBBECIgQRAiIEEQINLYYgvay7/KTJHj3ok/drCg+aK6EYDoKIgRECCIERAgiBEQIIgRECN/FsrUU3d78LPLUuwfW82ZNVa/LRk9gOgqIEEQIiBBECIgQRAiIEDoUXTHT7e49PYsMWnC3pcJb2QXvu5Z35OVD6koIpqMgQkCEIEJAhCBCQIQgQqCx/RUzg24cElS40UvqgaWKrInpeSVQcK2PKyGYjgIiBBECIgQRAiIEEQIiBBEC/5jt1ASuhCBCQIQgQkCEIEJAhPB1jmxv8ef3z+v6vyGD95p/Loa3df1U9/M3r/587jmuPvDpZ8HnfRC8ZXyyD+/V4L86eJ/dp6ejt0PWp6HfyHn+0/8Ib39A3H4QPy3t+KN4+Z3w6Uj9DmLnn9ZD+z1ZO79E3x3kboEug2/7TnhtEDdmj2+fqNydDQd/5u0R/p43j581d3/z6uAfz8jHi8DTD7KzA3V3qGfHMPhhevvA41c5HR66Er532nA7G3k6gXnjROX2Bx7/mRtHeGFudncMd1+hb68Jj6fjqxN0dx54YQzvDvjsdLrNo756OtpgDvP0PwXfpAtzuWvzpVMHH5xh7j7XtTG8NkW8HLzvhKenoz/XhAbj9d6naPwGv/rm/Dh0fc7BLhzVbbo/L/PID7n2KN8Jp1Nzs5bnwfFvONnH/+rgH7/pOe0om44+/b608buKN2ZzalZ25AgPHvypV3HkH7/6zvmWITp+qNcOI+/gP9XO/6yP9/D4K8fgLzM3Ln1Pf70ZOcLjB//qt6Pbx7/9qO3fjl54v679XvrsL4Qjz/WlEdre4qs+rV2Xev9OyMeTX4f+As9y2q6oFqayAAAAAElFTkSuQmCC) +![Answer](https://cdn.strawpoll.com/images/polls/qr/YVyPv9qP4gN.png) -[Answer link](https://toreply.univ-lille.fr/reponse_616) _Key: bh_ +[Answer link](https://strawpoll.com/YVyPv9qP4gN) ## Quizz @@ -96,9 +98,9 @@ Cite some V's of Big Data (multiple choices): - Answer D: Voldemort - Answer E: Variety -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAG/ElEQVR42u3d0ZLiNhAFUDvj///jFHmgamoCxjZuWi3hc562NgtIwhcJ0tWeb7fbBNRZpmma5/n7Jpb64bK7YtuvXrjgtcsSGdhXXqX3if/jcwhqCSEIIQghIIQghIAQghACQgjXs+z+i27r2oIlFNsPT511YT1N6pMXzmvoq9ROCI6jIISAEIIQAkIIQggIIQgh0NgSfPy4ZRCRJw8OrLAXS2Tku8Putg1M51epnRAcR0EIASEEIQSEEIQQEEIQQqCx5bIz77aKotsONIXFOnZCQAhBCAEhBCEEhBCEEBBCEEJACOF7XLdsLVIdFqzASu0xtT3yyLy67eNkJwSEEIQQEEIQQkAIQQgBIQQhBN4SrZgZt3tPYcek1CXNe0fGfa87H7mdEIQQhBAQQhBCQAhBCAEhBCEEGtuvmLlmZ5Fgt5W8Zi2pL10462BRy9BXqZ0QhBCEEBBCEEJACEEIASEEIQQam8dtHJK7LpkVGJE1H7dsxZVmJwQhBIQQhBAQQhBCQAhBCAEhBCEEpulettZtk5y8dkmpA5uGLeDquRfTFzewshOC4ygIISCEIISAEIIQAkIIQgg0tl8xk1q2klc7EiyDSH3ynmt9Ute8al6FxTpHntxOCI6jIISAEIIQAkIIQggIIQgh0Niy+y8K62kiD++5DUy39TTBt6PbcpzOq77shOA4CkIICCEIISCEIISAEIIQAkII1zLX3qarsBapcOKDluOl1tPV3nct9UrbfXI7ITiOghACQghCCAghCCEghCCEQGPLNOyduoKlId3WfxS2gSpsflV4me0uaWqFk50QHEdBCC0BCCEIISCEIISAEIIQAu3NqZUlg/Zx6XletRP/SrV3dNNjBhxHQQgtAQghCCEghCCEgBCCEALtFfeYiTQ1Kewxc81ynHFv4NX5otkJwXEUhBAQQhBCQAhBCAEhBCEEhBCuZZlK78W1La+obcq8K9uu1JfWXKurJdXoCRxHASEEIQSEEIQQEEIQQkAIoUO5jZ6CtQjbD0+tzyjsIhVZsdpZF448dWCpy2InBMdREEJLAEIIQggIIQghIIQghEB7S/Dxhf1pgpUKqbdGG3dZqoY96J3PPjJyOyE4joIQAkIIQggIIQghIIQghEBj8+12y26hkTX0jmssCm/51OekpmHrhOyE4DgKCCEIISCEIISAEIIQAkIIQgh8WLTRU2GNVWoVVeEdyFJnnTrsyNgK383Utl1H5mUnBMdREEJACEEIASEEIQSEEIQQaGzpeXCplQrZZRAlsy58O4J6vu9a9sVgJwTHURBCQAhBCAEhBCEEhBCEEGhsHvqeUqGZj3lDuHHn1W1Tn92Xzm5gYycEx1EQQkAIQQgBIQQhBIQQhBAQQriWZSrtHZSntu7sW29+1u2SRhat/JZvdkJwHAUhBIQQhBAQQhBCQAhBCIHG9m+N1m3Lo2B5R+G8Uss7Ctek57F1O2uNnsBxFITQEoAQghACQghCCAghCCHQ3hJ8/KBFEtOwt0YL3scrtflN5CZh39re5si87ITgOApCCAghCCEghCCEgBCCEAKNLZZgVbCeJl5FkaTw7kXf2qwovuB2QnAcBSEEhBCEEBBCEEJACEEIASGEa1G2ti5YdxYp4EqtooqU4wV7TA36Xk+xFlXK1sBxFBBCEEJACEEIASEEIQSEEDoUrZjptntPUGHZSuqCFzY1ClaW5I08WP8UH5idEBxHQQgBIQQhBIQQhBAQQhBCoLH9iplBG4eMK7W9TapITUx52UrSvOyE4DgKCCEIISCEIISAEIIQAkIIQgg8mr+1UxPYCQEhBCEEhBCEEBBCEELgpSPtLX5+/3y7/WvJ4LPm+2b4N1331N3/5tWf33uNsw9c/Sz4vg+Cj6xP9vBeLf7G4Hfn1fnEWx9H/y50n4Z+q+b5p/8V3s7J3w/i1XxuRPfcP7jid8LVRfldeh9aqZ8vfS7sw5v+kEMJbPed8Nz1sXF6/PiXzId39OBz/h3h79X2/Fnz8DevBv98VT1vHasfZO8u1MNQ313Dtz5M768yzz+r30dW98bd/7Q6BTvh/qpFzjCrB5hXx5vgax1/zo0RnjibPYzh4ZL9e9U+X/2v8rB7Djyxhkc2seyjkCPV1nE09Qxz5HjT7Cy3McLTZ7PPnjB3X+vcGr475Y9/aErgznH09+zR5ptGn8927uVWl67PSy1ydn04nX5qDeNP+FXfCdsc0z+44q++wrUf/PM3PdfWkVWyMaYcRzd+Q1v9reKDsXnrVHZkhAcH/9YsjvzjV985P7JEJ74Qykzu8Wr7f9bH8/D8k2Pwx8yNrW/1583ICI8P/tWvo9vj337U9q+jJ96vc79Lv/uD8JHVkOr/rYP2Fpe6Glz3vX8n5GrfyujBf9rN0eKYKGZ1AAAAAElFTkSuQmCC) +![Answer](https://cdn.strawpoll.com/images/polls/qr/6QnMQkd1Pne.png) -[Answer link](https://toreply.univ-lille.fr/reponse_8114) _Key: zv_ +[Answer link](https://strawpoll.com/6QnMQkd1Pne) # Legacy “Big Data” ecosystem @@ -165,7 +167,7 @@ Cite some V's of Big Data (multiple choices): ## Quizz -Which technology is the most representative of the Big Data world? +Which technology is (was?) the most representative of the Big Data world? - Answer A: Spark - Answer B: Elasticsearch @@ -173,9 +175,9 @@ Which technology is the most representative of the Big Data world? - Answer D: Tensorflow - Answer E: MPI (Message Passing Interface) -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAGz0lEQVR42u3d3ZKqOhAGUDnD+7/xKc6FVZZH5UebphNY62pq9igh8pHgTnWGaZpuQJ3xdrsNw3C+EwveXJb7pOU7V+TTTO204KFPeZXeT/wf9yGoJYQghCCEgBCCEAJCCEIICCFcz7j6F82uDgkuoUhdgVG4bCXy8mCfLB86tcO7vkqNhGA6CkIICCEIISCEIISAEIIQAgcbg6/vdxlEZHlH6sqSYIdHziu45KXZMjCNX6VGQjAdBSEEhBCEEBBCEEJACEEIgYONlz3zTrcQSl1FFNyLqt+trIyEIISAEIIQAkIIQggIIQghIIQghMBxrrtsrdlqS/2eNUZCEEJACEEIASEEIQSEEIQQEELoQnTFTL/VewqrEhWup4mcdb+fdeMtNxKCEIIQAkIIQggIIQghIIQghMDB1lfMqCyyu8iylcLyNqmHDi5q6foqNRKCEIIQAkIIQggIIQghIIQghMDBhn4Lh3Tc6YHlHaufV2HtnMJDGwkBIQQhBIQQhBAQQhBCQAhBCAEhhG6Mt4aL5BRuIZba8mbPOrjuLLXleVWkygtYGQnBdBSEEBBCEEJACEEIASEEIQQONkzTlFefJ3V5h5pFu/d5v/uTNbtYZ8ubGwnBdBSEEBBCEEJACEEIASEEIQQOtr5iZlnhspVgw/p987JrJVaLJfXoqeu6sj8RIyGYjoIQAkIIQggIIQghIIQghIAQwrVEl62larbGVMvnVbiFWFWf1LZNoScwHQWEEIQQEEIQQkAIQQgBIYTuDC3XLDproadOSzn1uyFc4yuBjIRgOgpCCAghCCEghCCEgBCCEAIHK94arXCJRr9VSU7ZsEK1l5kaM2A6CkKoC0AIQQgBIQQhBIQQhBA43lh7+GuuLCk868LVUYUtb/YKNxKC6SgghCCEIISAEIIQAkIIQggIIVzRUFupKW8V1ZYCO2WdftJd2cr3GEtqduTQCj2B6SgghCCEgBCCEAJCCEIICCE0KLfQU2HNovL9rn5+89QyUIWFnmrrREUaln2lGQnBdBSEEBBCEEJACEEIASEEIQQONkzTFFkQkFpjJru2R0nDmr4a6orEdLrz2S7dYiQE01EQQkAIQQgBIQQhBIQQhBA42Nhy4/otiFLY8rylJ8027Nb5GiYjIQghCCEghCCEgBCCEAJCCEIICCFcS9Nbo521nlJqAavCQ+dtCBdsW7DGVHZNMCMhmI6CEAJCCEIICCEIISCEIITAwYoLPeVtlxXclS345lU9tnpezZ51ahmo4Jtn75NnJATTURBCQAhBCAEhBCEEhBCEEDjYcNY6LutnXrdT1zU3Ccted5J36OwCNkZCMB0FIQSEEIQQEEIQQkAIQQgBIYRrGW/Jq6iqBJdBpa47S33z1DVWhfJWvZVv+WYkBNNREEJACEEIASEEIQSEEIQQONj61midFg5KPa9gnxRu+ZZabanTAlblV7iREExHQQgBIQQhBIQQhBAQQhBC4GBj8PX97vJVuFNX5KxTOzzYJ5GW91veJn4NGwnBdBSEEBBCEEJACEEIASEEIQQONuqCj1LX0xSWt8nrk9W2nbVYUbzTjIRgOgpCCAghCCEghCCEgBCCEAJCCNdi2dpnLW9+VnXo4K5s/X7WCj2B6SgghCCEgBCCEAJCCEIICCGcTHTFTLPVe1oW6bRgtaXCreyCtbPyWl7epUZCMB0FIQSEEIQQEEIQQkAIQQiBg62vmOm0cEhQ6tZoES0vUYp0WssrgbIvBiMhmI6CEAJCCEIICCEIISCEIISAEMK1DCo1gZEQhBAQQhBCQAhBCAEhhMvZUt7i7/HzNP2ry2Bfw30wfE7XPXX338z9/N0xfn3hx3vB+W4Eu/RPVTsXGt/LebUyHX0e7trU9Wc5DH/t9/D2CdHyL3u5olp8JvzYa48x0F0t9f7SeMdKYPEz4W/Xx8LscfeHzJePfON7PrfwcYt5v9e8/Gau8e+X3fMcfnU6t72jXpr6bR/+cDN9P+LcCW75J2ZHwn177fGxvX9yz/+0yxGf33D7ey60cOOrng/00oaXR+jHFf9xxJvLw3JHDcPfD3340uAzPa+eZDqa9Pi+MKrsksMf5nILLfz2VXOND84wV4/1Wx8GT5n06eh9TDhgCrHvIQ6e88zNvt67rs1r9+cvt18GYcnJeib8am525HWw/Vad3f65xr8/6Z3gYt3+vxG0NR39+Ly08F3FjrH5ala2pYUbG//VWWz547lnzl26yPclrVn5z/p4Ht6/cgx+mbkw9H38ejPSwu2Nn/t2dLn9y69a/nb0h88r+L30cld81Rv8r2OVt7jURMuUsvVnQk5P/Br0H0isqsH8XFhdAAAAAElFTkSuQmCC) +![Answer](https://cdn.strawpoll.com/images/polls/qr/61gD9MxPOZw.png) -[Answer link](https://toreply.univ-lille.fr/reponse_411) _Key: ge_ +[Answer link](https://strawpoll.com/61gD9MxPOZw) # Big Data use cases @@ -259,9 +261,9 @@ What is the typical volumes of scientific Datasets (multiple choices)? - Answer D: PBs - Answer E: EBs -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAG9ElEQVR42u3d3Q6bOBAG0LDl/d+4oheRomx+gGQYe0zOuaq6TTCGD5vsyJ6WZbkA/cyXy2WapvOdWPDhst4nlZ9ckauZ2mnBQ5/yLr2e+H+eQ9CXEIIQghACQghCCAghCCEghPB75s1/UbY6JFhCkVqB0bFsJfLxYJ+sHzq1w4e+S42EYDoKQggIIQghIIQghIAQghACjc3Bz49bBhEp70itLAl2eOS8giUvZZeBKX6XGgnBdBSEEBBCEEJACEEIASEEIQQam3/2zAfdQii1iii4F9W4W1kZCUEIASEEIQSEEIQQEEIQQkAIQQiBdn63bK3sakvjnjVGQhBCQAhBCAEhBCEEhBCEEBBCGEK0Ymbc1Xs6rkrUsZ4mctbjXuviLTcSghCCEAJCCEIICCEIISCEIIRAY9sVM1YWOVykbKXj8japhw4WtQx9lxoJQQhBCAEhBCEEhBCEEBBCEEKgsWnchUMG7vRAecfm9eq4dk7HQxsJASEEIQSEEIQQEEIQQkAIQQgBIYRhzJfCi+R03EIsteVlzzpYd5ba8rxVpLovYGUkBNNREEJACEEIASEEIQSEEIQQaGxaliV13aGNw6ctDVR5H6/cK1r1aqYqW6yz58uNhGA6CkIICCEIISCEIISAEIIQAo3Nm/8itWwl9eMddSxbyavmCRYhle3S1HoaIyGYjgJCCEIICCEIISCEIISAEIIQAo/m4OdTK8s6VsylfvlZF7BKLdYru0+ehZ7AdBQQQhBCQAhBCAEhBCEEhBCGE62YKbu2T6rNs85reeoyUONuCJd31pf8ZbuMhGA6CkIICCEIISCEIISAEIIQAo1Ny7KULe/IPfNAdUjqQi+pyjaso75VX9aYAdNREEJdAEIIQggIIQghIIQghEB7uWvMlC0N2dRxt6lxV9ZJNWg1jzVmwHQUEEIQQkAIQQgBIQQhBIQQhBB4NF9i5UjBzbTyNp2qXOWUutVW9j5evfo8r2guuGxX/MuNhGA6CkIICCEIISCEIISAEIIQAo3Nl3DVS0THYp2yyu7KFuzSslckWIQUPy8jIZiOghACQghCCAghCCEghCCEQGPTsixnLT3ZOPPAWaeuSlK2T4LGXRMolTVmwHQUhFAXgBCCEAJCCEIICCEIIdDeHPx8arVNxwVRUrslteV5pSe/uWeTkRBMRwEhBCEEhBCEEBBCEEJACEEIgYPNfQ8/6K5sHTfTChZ/pR46tUvzOjx1t7k952UkBNNREEJACEEIASEEIQSEEIQQaCx3a7TUGouOtSPBtnU8dMceO+vmZ/GraSQE01EQQkAIQQgBIQQhBIQQhBBobBp6T6nQmVddlSSo7AUtW0WUumLQnlvFSAimoyCEgBCCEAJCCEIICCEIISCE8Fu2F3oaVLAWKfjloUty0gWRUq9Xap1gdiWgkRBMR0EIASEEIQSEEIQQEEIQQqCxefNfDLpwUPC8UstWOhbrpK62NOgCVt3vcCMhmI6CEAJCCEIICCEIISCEIIRAY3Pw8+Pu8lV2KZeypTyblyPS8sor62RXOBkJwXQUhBAQQhBCQAhBCAEhBCEEGpt1QXuRSqCOC6JsFo6st+2sixXFO81ICKajIISAEIIQAkIIQggIIQghIITwW5StvdZxk7Cyh95sWOXFmnqdl4WewHQUEEIQQkAIQQgBIQQhBIQQCopWzJRdvaeySKcFV1vquJVdcN+1vJZ371IjIZiOghACQghCCAghCCEghCCEQGPbFTODLhxyVpVLlCI1MZUrgYK1PkZCMB0FhBCEEBBCEEJACEEIASEEIQT+Z7JSExgJQQgBIQQhBIQQhBAQQvg5e5a3+HP787L81WVwrOk6GN6n65q669+8+/Nnx/j2gy+fBed7EBzSP3kNe/bubvHsDo2E0/SneE8ty99394SY5fX5eizfXZGHZ3f9u6vKO+HLDr3vR4+0vHt9xI7dk8CHaRRfvhN+d3+szB4Pn6g8XOCd33nfwtt98/ysefibd41/vsmeb76XD7JPO+qhqZ/2YeRhujNd17bdRj+P770j4bGPq1u/P88h7//TIUe8/8L937nSwp2fuj/QQxsebr77u/b5Xnx3d6531PUW/7QPHxqcN6M2gfpyOpr0/rMyqhySwy/mcistjI8Dh8wwN4/1XR9+espfj5+HP2p/ZTp6m0X0epeo8G3fHe5l19V8/Df4cfv5CeK3mQ/eCdv8AnngxXj3Cte+8c9veue47cwnh5yOvnxfWvmt4sDYfDQr29PCnY3/6Cz2/ON375yHdNEX78DxG0CM1/p5/X/Wx/Pw/JNj8MfMlaHv5c+bkRbub/y7X0fX27/+qfVfR7+4Xp/+Lr3+zV80ntc9aXmLn3paG5eqvxNyeuJX0D/f8dGlf71iDAAAAABJRU5ErkJggg==) +![Answer](https://cdn.strawpoll.com/images/polls/qr/BJnXVEWkXZv.png) -[Answer link](https://toreply.univ-lille.fr/reponse_718) _Key: fr_ +[Answer link](https://strawpoll.com/BJnXVEWkXZv) # Big Data to Machine Learning @@ -295,7 +297,7 @@ What is the typical volumes of scientific Datasets (multiple choices)? Are Big Data and Machine Learning the same? -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAHC0lEQVR42u3d3Y6jOBAGUNjm/d94xV601MrmxxCKouzkHO3FaqY7GMOHTaZkz+u6TkCdZZqmeZ4/78SCD5d2n/T85IpczdROCx76I+/S3xP/x3MIagkhCCEIISCEIISAEIIQAkII32fZ/Iluq0OCJRSpFRiFZSuRXw/2SfvQqR0+9F1qJATTURBCQAhBCAEhBCEEhBCEELjYEvz9ccsgIuUdqZUlwQ6PnFew5KXbZWA6v0uNhGA6CkIICCEIISCEIISAEIIQAhdbvvbMB91CKLWKKLgX1bhbWRkJQQgBIQQhBIQQhBAQQhBCQAhBCIHrfG/ZWrerLY171hgJQQgBIQQhBIQQhBAQQhBCQAhhCNGKmXFX7ylclaiwniZy1uNe685bbiQEIQQhBIQQhBAQQhBCQAhBCIGLbVfMWFnkdJGylcLlbVIPHSxqGfouNRKCEIIQAkIIQggIIQghIIQghMDF5nEXDhm40wPlHZvXq3DtnMJDGwkBIQQhBIQQhBAQQhBCQAhBCAEhhGEsU8eL5BRuIZba8m7POlh3ltryvFWkyhewMhKC6SgIISCEIISAEIIQAkIIQghcbF7XtbD0pNvFf8Zdsyh1Fam8Q6feSIXFOns+3EgIpqMghIAQghACQghCCAghCCFwsWXzJ7IX2Dj84UGFm4SlNizvvIJrsXTbpan1NEZCMB0FhBCEEBBCEEJACEEIASEEIQTuzZtFN4NunLZ95mMub1W4j1fPVYSFbbPQE5iOAkIIQggIIQghIIQghIAQwnCWadgldAp3LyssM0pdBmrcDeFS79LsZbuMhGA6CkIICCEIISCEIISAEIIQAhdbag8fKcIoLOBIXeil2w7vdrWhYMtrd3SzxgyYjoIQ6gIQQhBCQAhBCAEhBCEErjen7oYTXIulcEGU1NVxUg+dp3aNmW6X7TESgukoIIQghIAQghACQghCCAghCCHwnrlwl6+gwsWUCmviUisBozdTx30eaXZqEaKFnsB0FIRQF4AQghACQghCCAghCCFwve2t0VILUwoXRIqcV2qNRbDD865XsGalfD2lww3LLkIyEoLpKAghIIQghIAQghACQghCCFxsXte1sIoirxahsJ5m6rg6JLXTUrv0U1ljBkxHQQh1AQghCCEghCCEgBCCEALXi64x01a7303eoVPbVrjl06ANm4YtUTISghACQghCCEIICCEIISCEIISAEMI3WmoPH1noadxKpUGL9Qo3hEu9kVJ3m9tzXkZCMB0FIQSEEIQQEEIQQkAIQQiBi21vjZZaLlC4KlHermzZH57XsGCXVl3rWvGraSQE01EQQkAIQQgBIQQhBIQQhBC42Dz0nlKhM68ravnOTcK67fDU4qo9C9gYCcF0FIQQEEIQQkAIQQgBIQQhBIQQvsv2Qk+DKlxjaird/Ow7r1ewdizvahoJwXQUEEIQQkAIQQgBIQQhBIQQOrRs/sSgCwcFz6tw47TUy9HteRXeh+V3uJEQTEdBCAEhBCEEhBCEEBBCEELgYkvw98fd5SvS8j37XSWddWqHB+tpIi3veWWd7AonIyGYjoIQAkIIQggIIQghIIQghMDFFl1wvbx6mlTBOqFPXawo3mlGQjAdBSEEhBCEEBBCEEJACEEIASGE76Js7bngakvdbn4WOfRmw3perKnqvCz0BKajgBCCEAJCCEIICCEIISCE0KFoxUy3q/cEdbvlW3C1pcLzCu67ltfy8i41EoLpKAghIIQghIAQghACQghCCFxsu2Jm0IVDggrrTiINqxWpiem5EihY62MkBNNRQAhBCAEhBCEEhBCEEBBCEELgf+ZPXakJjISAEIIQAkIIQggIIQgh8NKe5S1+/v5/Xf/VZXCu+XcwvE3Xb+p+/+TV/793jKO/+PRZ8HkPglP6J69hj/6a2nhAe3a/PR191d39GPpCzvNP/z38tM/v/nv67Lh9Rm/+Fa13wqc99deJ3T6tP8Dj/T3c6H0btsZfcfCd8Nj90Zg9nj5RubvAOz/ztoV/983js+buT141/vEma4wPkY66a+q7fRh5mHoQ546E5z6ubmcjd595+kTl9gP3f2ajhTt/6/ZAd224e4W+HRMe7+BX93S7o+b550Af3jX4rKHv6fMicqzvnY6mTlT2XMjL5nKNFh6+C8+dYW4e61gfvnvK7THw9lVl/1/Rmo7+jgkXPLfOPcTFD9pXb86PXdfnzXful9uNT5O9g++Eb83NrrwP9t8f2e1/1fjHN73PGAQaCWycoAGweDq6/21hOvu7+7dmZe++zzQa/9ZZ7PnhV++cp3TRgXfgnS+ZT5+Gvs7Z7uf2P9bH8/D4lWPwy8zG0Pf0681IC/c3/tW3o+32t3+r/e3ogev17vfS+0M4+cf6SFgsb/FVT2vjUu/vhHw88evQf2qc/ngzQXEaAAAAAElFTkSuQmCC) +![Answer](https://cdn.strawpoll.com/images/polls/qr/GPgVYMaJzna.png) -[Answer link](https://toreply.univ-lille.fr/reponse_799) _Key: dq_ +[Answer link](https://strawpoll.com/GPgVYMaJzna) diff --git a/src/02_Big_Data_Platforms.md b/src/02_Big_Data_Platforms.md index 2b4f177..f7b6e16 100644 --- a/src/02_Big_Data_Platforms.md +++ b/src/02_Big_Data_Platforms.md @@ -1,7 +1,7 @@ --- title: Big Data Platforms, Hadoop and beyond author: Guillaume Eynard-Bontemps, CNES (Centre National d'Etudes Spatiales - French Space Agency) -date: 2025 +date: 2026 --- # Hadoop @@ -86,9 +86,9 @@ What are the **two** building blocks of Hadoop ecosystem (multiple choices)? - Answer C: Map Reduce - Answer D: Servers -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAG5ElEQVR42u3d2Y7jNhBAUSut///jwPNgwDC8aHGpWKR1DvIQTKbbElvXlDsENV2v1wtQZ75cLtM0/d6JBd9clsek53euyE8zddCCL/2TV+ntxP/zPgS1RAgiBBECIgQRAiIEEQIihPOZV/9Gt6tDgksoUldgFC5biXx5cEyWXzp1wIe+Ss2E4HYURAiIEEQIiBBECIgQRAg0Nge/ftxlEJHlHakrS4IDHjmv4JKXbreB6fwqNROC21EQISBCECEgQhAhIEIQIdDYfNozH/QRQqmriILPohr3UVZmQhAhIEIQISBCECEgQhAhIEIQIdDOeZetdbvb0rhnjZkQRAiIEEQIiBBECIgQRAiIEIYQXTEz7u49hbsSFa6niZz1uD/rzo/cTAgiBBECIgQRAiIEEQIiBBECja2vmLGzyOEiy1YKt7dJfengopahr1IzIYgQRAiIEEQIiBBECIgQRAg0No27ccjAgx5Y3rH68yrcO6fwpc2EgAhBhIAIQYSACEGEgAhBhIAIYRjzpeNNcgofIZZ65N2edXDdWeqR5+0iVb6BlZkQ3I6CCAERgggBEYIIARGCCIHGpuv1Wrj0pPBhWivjMuyeRam7SOW9dOGFVP7NzYTgdhRECIgQRAiIEEQIiBBECDQ2BR+1larbPWZSB63b5TjBvVhSXz3y0rXnZSYEt6MgQkMAIgQRAiIEEQIiBBECIoTTWd/oqXDtWOFapHOe16BjUntsNnoCt6OACEGEgAhBhIAIQYSACGE4U+2eRXkLHXo+r0G3chr3gXCdrwQyE4LbURAhIEIQISBCECEgQhAh0Fh0j5mV797xriSpRz7oQ90KD6xQ7YVkjxlwOwoiNAQgQhAhIEIQISBCECHQXu5TmVL3Yhl3Q5TImBSqHdLC7YjMhOB2FBAhiBAQIYgQECGIEBAhiBA40tzzwaU+7yp1y6O8b97zU9kG3dcreKnEv7mZENyOgggBEYIIARGCCAERggiBxuZL5vKOX33AWKpuVwIF16x0u/VW+SIkMyG4HQURAiIEEQIiBBECIgQRAo1FH42We3C97jEz7nPXIgMedM71T1uGxUwIbkdBhIAIQYSACEGEgAhBhEBj0acydfsMoNoNUQq3cslbetLtgV0GX8NkJgQRgggBEYIIARGCCAERgggBEcK5RJetFe7eM+5KpciRBzewikh96dQ1cZ3vCWYmBLejIEJAhCBCQIQgQkCEIEKgsa43elr+5sHlOIM+Gm31pbvd6KnqsOPfPPtiMBOC21EQISBCECEgQhAhIEIQIdDYNPQzpUJn/otPL7t0vPXOuE/Ry97AxkwIbkdBhIAIQYSACEGEgAhBhIAI4VzmS/Iqqio9L8dLfZ7cT/40L5mr3sof+WYmBLejIEJAhCBCQIQgQkCEIEKgsfVHow26cVDwvLp9UlfqrkS1P5Gq67D8CjcTgttRECEgQhAhIEIQISBCECHQ2Bz8+nGf8pX9vKuksy7cgWb1pCJH3vPWONlP0TMTgttRECEgQhAhIEIQISBCECHQ2GwI2st+yk+S4DqhX92sKD5oZkJwOwoiBEQIIgRECCIERAgiBEQI52LZ2ns976dU9dLBp7J1K/W8bPQEbkcBEYIIARGCCAERgggBEUKHoitmut29J1VwjUVk0IK7LRU+yi44JnlHXj6kZkJwOwoiBEQIIgRECCIERAgiBBpbXzEz6MYhqQrHpOclSpE1MT2vBMpb/2QmBLejgAhBhCBCQIQgQkCEIEJAhHBG0zl3agIzISBCECEgQhAhiNAQgAjh1LZsb/F3//fr9X9DBseabpPhY1236m5/8unf973Gt1/49r3g994IDhmf9kf4+O68/F+9d2+6HX0a0A4N/YOcpr/+R3jLrdBrlrd/nv7a438d4urq5TPh25G6D2L/79bjul+soxT4esyPsT1dKjo84DPhd9fHwt3j4Tcqy/dFW47wft28vtc8/cmng3+9yBbmh8hAPR3q3jHc+2a6MSHv0QfMhMe+XT3ejTx9z8NvVJbvi744wo1f9fhCT8fw9BH6cU54vUw/XbjLAzVNf1+M4dMBH3un/WnqMwfuuB1N+hC/MKsc0uEX93ILR7j3qzberR0yhm8nxl1juPeUd43/6+Hd+jQ97rgdvc0JDd63jn2Jxm+0nz45vw5dnxdf0lHdRyCj83N9Jtx1b9bDdfDpI1z7g3/9pHfmScAEWHw7uvCR4O3vKo79fHLsEW48+F1nseUvf/rMeeBHuGO/z8K7od/ZrA/m8v+sj/fw+ivH4C8zF6a+t7/ejBzh9oP/9NvR5eNf/qrl345+8fP64vfSC0fof9YfFqHtLU71bm1e6v0zIT9Pfh36B8Ma2rE/VRWMAAAAAElFTkSuQmCC) +![Answer](https://cdn.strawpoll.com/images/polls/qr/ajnE1M3QxnW.png) -[Answer link](https://toreply.univ-lille.fr/reponse_139) _Key: uw_ +[Answer link](https://strawpoll.com/ajnE1M3QxnW) ## Map Reduce exercise @@ -173,9 +173,9 @@ What means HDFS? - Answer C: Hadoop Distributed File System - Answer D: Hadoop Delayed Functional Services -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAG/UlEQVR42u3d267aOhQFUNKd///jKn1A2qJJyG1l+QJjPB2dFuIYz9jQJXuYpukB1DM+Ho9hGD7vxoIPl+0+afnJFfk0UzsteOmPHKXPG//jOQR1CSEIIQghIIQghIAQghACQgjfZ9z9G81WhwRLKFIrMCqWrUReHuyT7UundnjXo9RMCJajIISAEIIQAkIIQggIIQghUNgYfH2/ZRCR8o7UypJgh0fuK1jy0uw2MI2PUjMhWI6CEAJCCEIICCEIISCEIIRAYePX3nmnRwilVhEFz6Lq9ygrMyEIISCEIISAEIIQAkIIQggIIQghUM73lq01u9tSv3eNmRCEEBBCEEJACEEIASEEIQSEELoQrZjpd/eeirsSVaynidx1v5914y03E4IQghACQghCCAghCCEghCCEQGH7FTN2FrldpGyl4vY2qZcOFrV0PUrNhCCEIISAEIIQAkIIQggIIQghUNjQ78YhHXd6oLxj9/OquHdOxUubCQEhBCEEhBCEEBBCEEJACEEIASGEboyPhjfJqXiEWGrLm73rYN1ZasvzdpGqvoGVmRAsR0EIASEEIQSEEIQQEEIQQqCwYZqmiqUnnR6m1fKeRRXvq9OBVP3NzYRgOQpCCAghCCEghCCEgBCCEAKFDf0WSVQUPJ8s+ObVxkpsL5bUq0cuXfe+zIRgOQpCqAtACEEIASEEIQSEEIQQEEL4OkOwAqvf3Zaa3ZUo2OyKR4jV6pO6bbPRE1iOAkIIQggIIQghIIQghIAQQndyN3pKLcepWMpTt8wocTT02exH55VAZkKwHAUhBIQQhBAQQhBCQAhBCIHChmmaOt1tZefGMvdiCd5Xsx3uoLsq49BMCJajIISAEIIQAkIIQggIIQghUNjQ73YpLR8S1Ox9ddonqd1Sfe8cMyFYjoIQAkIIQggIIQghIIQghIAQwncZg6/vd8+iTjd6arnMsNNCwuBQib+5mRAsR0EIASEEIQSEEIQQEEIQQqCw8ZFZwFG3FiFPat1Js5VAwbuuvp/S5YZlFyGZCcFyFIQQEEIQQkAIQQgBIQQhBAobpmn64EOnsnotsxKo5fvKGwzNHgiXzR4zYDkKQqgLQAhBCAEhBCEEhBCEEChv/1SmTs/xSd04JPjmqR3ebP1Tp2c2mQnBchQQQhBCQAhBCAEhBCEEhBCEELjZ2HLjUg9Oq1jo1Ol9pV46tSau4mlzR+7LTAiWoyCEgBCCEAJCCEIICCEIIVBYbsVMai1Catua3TgodY+p4KVTh0rFN88eKmZCsBwFIQSEEIQQEEIQQkAIQQiBwoauz5QK3fknnl72aLjWp2KJUvDS2RvYmAnBchSEEBBCEEJACEEIASEEIQSEEL7LME1TahVVLRU3RHokH372kerWjuV9mmZCsBwFhBCEEBBCEEJACEEIASGEBu0fjdbpxkGp95V61Faw2TawauedzYRgOQoIIQghIIQghIAQghACQghtGoOv7/eUr4ondUXuOrXDg30SaXnLO+tkFyGZCcFyFIQQEEIQQkAIQQgBIQQhBAobdUF5kXKciqU8u4Uj22371M2K4p1mJgTLURBCQAhBCAEhBCEEhBCEEBBC+C7K1tYFtzzq9HyyYN1Zy5s1tfxpmgnBchSEEBBCEEJACEEIASEEIQQKi1bMNLt7T6pgaUik04K7LVU8yq5iEVLjXWomBMtREEJACEEIASEEIQSEEIQQKGy/YqbTjUOCKtadRBpWV6QmpuVKoGCtj5kQLEcBIQQhBIQQhBAQQhBCQAhBCIH/DN+5UxOYCQEhBCEEhBCEEIRQF4AQwlc7sr3Fz+9/T9NfXQb3Gp6T4Wu6nql7/p93/33uGldfuPos+LwHwS39k9ewmd92zv501n7P7tMz4TD8NN5T0/R3dUyIWYGe372jYfh5HUKzZ3f7o6uV74Qbj71nP3qk5Y3y7jp21ubXtdJsqLz+ERe/E14bHxurx9sXKtvroiMt/B03y2fN8pG/eqHlIFsOvtUH2dmOmjX1bB9ee5juXssz+oaZ8N7H1e9HslxDvv7RLVd8fcPj77nRwoOver3QrA2zr9Cvc8JymO4O69Wbei7wzvbhrMHHX7VxreUqabVzOLQcTfr+szGr3JLDC2u5jRaefdW7xgdXmLvXutaHZ2/59Zny7lqr+fRt8OJy9DknFHhu3XuJwg/ad9+cl13X5uBLatVvD5zNOfPvhGV+gbzx43n3Fa5845ff9D5gEoj8o5QQ1lyObnwlWP2t4sbYnFqVHWnhwcafuosjf/ndd85buuh4UzeaMXufjaeh32z2P5Ttf6yP52H5k2Pwx8yNqW/1581IC483/t2vo9vt337V9q+jFz6va79LH/xB2D/WXw+L7S2+6mltXmr9OyEfT/wa9A+NtvKL72DKJwAAAABJRU5ErkJggg==) +![Answer](https://cdn.strawpoll.com/images/polls/qr/40Zm4E9Alga.png) -[Answer link](https://toreply.univ-lille.fr/reponse_539) _Key: vn_ +[Answer link](https://strawpoll.com/40Zm4E9Alga) # Map Reduce @@ -234,9 +234,9 @@ What is the magical hidden step of distributed Map Reduce? - Answer C: Shuffle - Answer D: Split -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAGyUlEQVR42u3d0a7aOhBAUdKT///jKn1AOqJAQmAy9jis9VT1FgiGTQzXcqZlWS5AP/Plcpmm6XxPLPjhsj0mlT+5Iq9m6qAFH/qU79LrE//jcwj6EiGIEEQIiBBECIgQRAiIEL7P/PJflF0dElxCkboCo+OylcjNg2Oy/dCpAz70u9SZEExHQYSACEGEgAhBhIAIQYRAY3Pw9uMug4gs70hdWRIc8MjzCi55KbsNTPF3qTMhmI6CCAERgggBEYIIARGCCIHG5q995oNeQih1FVHwWlTjXsrKmRBECIgQRAiIEEQIiBBECIgQRAi0873L1srutjTus8aZEEQIiBBECIgQRAiIEEQIiBCGEF0xM+7uPR13Jeq4nibyrMd9rYsfuTMhiBBECIgQRAiIEEQIiBBECDT2esWMnUUOF1m20nF7m9SHDi5qGfpd6kwIIgQRAiIEEQIiBBECIgQRAo1N424cMvCgB5Z3vHy9Ou6d0/GhnQkBEYIIARGCCAERgggBEYIIARHCMOZL4U1yOl5CLPXIyz7r4Lqz1CPP20Wq+wZWzoRgOgoiBEQIIgRECCIERAgiBBqbOu78E2TPosPHfNzrk5VdrLPnzp0JwXQURAiIEEQIiBBECIgQRAg0dto9ZoKXEAvquGwlbzVPcC+WskOaup7GmRBMRwERgggBEYIIARGCCAERggiBe9OyLB0XBGVfdOrzcam6RVXHIa28bVfHY7PRE5iOAiIEEQIiBBECIgQRAiKE4UypKxU67rZU+XmVvbLaWS8IV3wlkDMhmI6CCAERgggBEYIIARGCCIHGptTFH0Pv/BG5847DMuiBddR3syJ7zIDpKIjQEIAIQYSACEGEgAhBhEB7nfeYidz5ttTtbca9IFRE3z1mym5H5EwIpqOACEGEgAhBhIAIQYSACEGEwHvmS9cFXGWvxVV2YVrlq7J1v8ZY0mFHHtpGT2A6CogQRAiIEEQIiBBECIgQCpqDtw+uz8hbi9DxoYN3nnr1so6ro7rvp/TxgWUvQnImBNNRECEgQhAhIEIQISBCECHQ2LQsy6AXCat83bVR3w39Nokpe0G4bPaYAdNREKEhABGCCAERgggBEYIIgfZe7zFTdi+W1D1mxt3KJW/pSdkDuwy+hsmZEEQIIgRECCIERAgiBEQIIgRECN8l99JowZt3XDGXugyq4/OKDHjHC8KlvpE6DqkzIZiOAiIEEYIIARGCCAERggiBLubKB+fqZR88r8igpV6LLu+ws+88e1icCcF0FEQIiBBECIgQRAiIEEQINDaddWXJ62d+0tUhZV/QsgMeXIQU38DGmRBMR0GEgAhBhIAIQYSACEGEgAjhu8yX5FVUvVRejhdc6LTtlK/mpfBV9OK3dSYE01EQISBCECEgQhAhIEIQIdDY60ujDbpxUOrzCq6xSL3kmw2s6tyzMyGYjgIiBBECIgQRAiIEEQIihJrm4O3HvcpX2b1Yyu5A8/LliBx55a1xUgfNmRBMR0GEhgBECCIERAgiBEQIIgTamw3BU6nLVrKv8pMkuLPOWTcrig+aMyGYjoIIARGCCAERgggBEYIIARHCd7Fs7bngBcaytwbq8tCpz7qj7q+mMyGYjoIIARGCCAERgggBEYIIgcaiK2bK7t6T+rw6bvQU3G2p46XsOi5CKj6kzoRgOgoiBEQIIgRECCIERAgiBBp7vWJm0I1DUnVc3lF5iVJkTUzllUDBtT7OhGA6CogQRAiIEEQIiBBECIgQRAj8ZzrrTk3gTAiIEEQIiBBECIgQRAis2rO9xc/vn5flryGDY03Xk+FtXdfqrn+z9uf3HuPTGz79LDjfB8Eh49P4CG8/mm+tvZF4fSacpp/ig7Usf9deeJk1OPiN2J7+s3FfrJ7fCTfGepp+fKqlfr6UHdjPWlLgwd8JP3t/bMweD/+SefeS77zP2yP8/YhZm3f9/s3awT++7W7n8Bvnw3cH6u5Q3x3Ddz9MH5/FnrvdeSv+OxMeO2q/L8njHPL2Px3yiLd3uP8+N45w561uH+juGO6+Qt++NR/f/Ws9bA/U9evDu2N4d8Am2+Wmo0kvycZZ5ZAOP5jLbRzhUR/2wRnmy8f6bAzffcoKbD0dvZ4TGkwhMj6Gu/9K8Th0Nd+Xxx6VAo//TtjmF8jUj+Hs4187+MdvevV/cFbgaaejT78vbfxWcWA2b83K9hzhzoN/61ns+cdr3zkPGaJDBlyBR36cbf/P+ngPjz85Bn/M3Dj1Pf15M3KE+w9+7dfR7ePfvtX2r6MfvF4f/C69/avv9q9K9SfnVSK0vcVXfa47g1X/Tsjpya+gf5iWvK0BQduFAAAAAElFTkSuQmCC) +![Answer](https://cdn.strawpoll.com/images/polls/qr/e2naXe5W0yB.png) -[Answer link](https://toreply.univ-lille.fr/reponse_171) _Key: nm_ +[Answer link](https://strawpoll.com/e2naXe5W0yB) # Datalakes @@ -289,9 +289,9 @@ What is the goal of a Datalake? - Answer B: Host any kind of Data, at any stages of processing - Answer C: Standardizing Data structure -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAHFElEQVR42u3dUa+jNhAGUOjy//9xRR8ibSNCDGQYbMM56kO1e5MYhw+bu6NhnOd5AOqZhmEYx/F+B5Z6cdmcsfKnV5zwutMSGdgtz9LXgf/jOgR1CSEIIQghIIQghIAQghACQgjPM23+RLN1bcESivLLU4+6Yj1N6ptXPK6uz1IrIdiOghACQghCCAghCCEghCCEwMWm4Ov7LYOIvHlwYBV7sURGvjnsZtvANH6WWgnBdhSEEBBCEEJACEEIASEEIQQuNj32yJutomi2A03FYh0rISCEIISAEIIQAkIIQggIIQghIIRwH88tW4tUhwUrsFJ7TJVHHjmuZvs4WQkBIQQhBIQQhBAQQhBCQAhBCIFDohUz/XbvqdgxKXVK876Rfr/rxkduJQQhBCEEhBCEEBBCEEJACEEIgYttV8w8s7NIsNtKXrOW1I+ueNTBopauz1IrIQghCCEghCCEgBCCEAJCCEIIXGzst3FI7rxkVmBE5rzfshVnmpUQhBAQQhBCQAhBCAEhBCEEhBCEEBiGV9las01y8tolpQ5s6LaAq+VeTDduYGUlBNtREEJACEEIASEEIQSEEIQQuFhuxUzLvX2arbFIlfqNNHtcFYt19ry5lRBsR0EIASEEIQSEEIQQEEIQQuBi09Dtw7RSyyBSNVtPE/myhobLcRo/kayEYDsKQggIIQghIIQghIAQghACQgjPMgVfHyxqq/i8q4iKDwkLPpUtVepxNVvrp9ET2I4CQghCCAghCCEghCCEgBBCd7YrZip272m2A1XqpFUsDUmd0oqCU5paCWQlBNtREEJTAEIIQggIIQghIIQghMD1xmDPkmYbvdR9jle733erz12rqO4T3fSYAdtREEJTAEIIQggIIQghIIQghMD1xnmeKzZ6ySvgqFsGUXHknU7pY4/LSgi2oyCEpgCEEIQQEEIQQkAIQQgBIYTH2X40Wmqjp7yauIrPNkt985YbWLVczVdrSjV6AttRQAhBCAEhBCEEhBCEEBBCaNCU+u4Vn7uW2mOq5QZWFac0OPJaUqfUSgi2o4AQghACQghCCAghCCEghNCg7YqZiq1cKra3KUvtndOs1GF3OienjNxKCLajIISAEIIQAkIIQggIIQghcLFoj5mKTy/qt4Cj0+MKftedPrPJSgi2o4AQghACQghCCAghCCEghCCEwMmiZWupjZ4ir63YgSp10io+xyu1d1ZqTVxqT7D4hFsJwXYUhBAQQhBCQAhBCAEhBCEELjbO85xXY7H98ZmFDnlaLu/o9Kj7FQ+IlRBsR0EIASEEIQSEEIQQEEIQQuBiY9fPlAod+U2f1FWx/qnZgQU/OruBjZUQbEdBCAEhBCEEhBCEEBBCEEJACOFZpuGmHXiCZVAVmxo1W1lW140f+WYlBNtREEJACEEIASEEIQSEEIQQuNi0+RPNdoIKlndE+vNULMdpuT9Vp72z6h61Rk9gOwpCaApACEEIASEEIQSEEIQQuN4UfH2/Dxhr9hFiwTkpH1dq85tIEdJd29vsOS4rIdiOghACQghCCAghCCEghCCEwMUmU7AqWE8Tr6JIUvHpRXdtVhSfcCsh2I6CEAJCCEIICCEIISCEIISAEMKzKFv7RWoBV2oVVaQcL9hjqlmpvbOUrYHtKCCEIISAEIIQAkIIQggIITQoWjHTbPeeVMGjTn0+Wd5HB8cWrCzJG3mw/ik+MCsh2I6CEAJCCEIICCEIISCEIITAxbYrZjptHNKvYH1GxRqmSE1M9bKVpOOyEoLtKCCEIISAEIIQAkIIQggIIQghsDQ+s1MTWAkBIQQhBIQQhBCE0BSAEMKj7Wlv8efv/8/zv6YMzjW+FsP3dL1S9/qTb/9/7DN+feHqteB+F4JT5id7eIvJf//D1Wt04drtsr6+HV2d06Z0/W2N45/2Z7h8gXi/EL9/Ke//7XlV+Q2ffk9YuLb9PYdct5KuL21O7OJL3xmbwqtW/4ppzyly7u7x9N3I5+X56Aj/nhyf15rFn3wb/OepWVgEIhO1GOrROTx0Md3zY8HbBDn8fyU8d2/wvuVYvOfpu5H3N9z/noUR/rA3W4xhcQv9fuH/POe+nYXliRrHPz/M4WLA8XvX1WGsTs7q/txedLkdTd3DRLY3p+/lftsUHRp8cIe5+Vm/zWFkH/g5pMX1ZTWHr5R+vpV7wvXt6GtNuGBSzv2Ii7/Fb3fOn1PX5kYrcnNx6LWFH36P7regPvee8NDe7OLzYP8GqcrgP+/0bnNuFW5lC3tpN3s1t6OFW4LV31WcGJtDu7I9I9w5+ENHseeHv91znjJFhyb820dvjnD1r/KOq+/LXPkf6+N5+PyVY/CXmUPxH4X3D37PCPcP/ttvR8vjL7+q/NvRH76vo7+XLtzxbr6hf6w/EBbtLR51Sbb4tH5PyO2JX4P+A5TzPYhFHujZAAAAAElFTkSuQmCC) +![Answer](https://cdn.strawpoll.com/images/polls/qr/Qrgew4vl2yp.png) -[Answer link](https://toreply.univ-lille.fr/reponse_2596) _Key: gj_ +[Answer link](https://strawpoll.com/Qrgew4vl2yp6) # Data pipelines and associated tools @@ -641,9 +641,9 @@ How Big Data processing differs from classical HPC (multiple choices)? - Answer D: It uses commodity hardware - Answer E: It is fault tolerant -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAHIUlEQVR42u3d3Q7iNhAG0KTL+79xRS+QtiiQH5iMxw7nXK3aBRwnX2zY0WS+3+8TUOc2TdM8z9c7sNSby+6MbX964YTXTktkYJe8Sh8H/o/7ENQSQhBCEEJACEEIASEEIQSEEH7PbfdvdFvXFiyh2H556lEX1tOkvnnhcQ19lVoJwXYUhBAQQhBCQAhBCAEhBCEEGrsFXz9uGUTkzYMDK+zFEhn57rC7bQPT+VVqJQTbURBCQAhBCAEhBCEEhBCEEGjs9rNH3m0VRbcdaAqLdayEgBCCEAJCCEIICCEIISCEIISAEMJ1/G7ZWqQ6LFiBldpjanvkkePqto+TlRAQQhBCQAhBCAEhBCEEhBCEEPhItGJm3O49hR2TUqc074yMe647H7mVEIQQhBAQQhBCQAhBCAEhBCEEGtuvmPnNziLBbit5zVpSP7rwqINFLUNfpVZCEEIQQkAIQQgBIQQhBIQQhBBobB63cUjuvGRWYETmfNyyFVealRCEEBBCEEJACEEIASEEIQSEEIQQmKZH2Vq3TXLy2iWlDmwatoCr515MF25gZSUE21EQQkAIQQgBIQQhBIQQhBBobL9iJrVcIFJFkVpP0+2j0YJSi3W6Pa7ai2H3za2EYDsKQggIIQghIIQghIAQghACjd2mzIKAzisVqia923qa4Nnsthyn8wvJSgi2oyCEgBCCEAJCCEIICCEIISCE8Fuij0brtiCo56K2vAmfMh9vllpPV/vctdqL3EoItqMghIAQghACQghCCAghCCHQWLTRU+Gj0a7aRaqwDVRetU2t4JSmVjhZCcF2FITQFIAQghACQghCCAghCCHQ3pzas6TwYVo9N0SpPN+9PnetUG0/IT1mwHYUhNAUgBCCEAJCCEIICCEIIdDeLfj6YAXGVQs4sruSlExpbZ3QoJeKHjNgOwoIIQghIIQghIAQghACQghCCCzN9/s974FY43Zb6nbkhb2zrnq6U6dUoyewHQWEEIQQEEIQQkAIQQgBIYQO3abw483yRAY27tPLInMyddzoqdszkvp0QCsh2I4CQghCCAghCCEghCCEgBBCh/Z7zHQrtdAn+OaFRUiD9gS66kPyjkyLlRBsR0EIASEEIQSEEIQQEEIQQqCx2+7fKHyE0PbLUzt/XLV2JPLm406plRAQQhBCQAhBCAEhBCEEhBCEEBBC6Mit9uMHraLq+aPzqvmCRYiFU1rYtuvIcVkJwXYUhBAQQhBCQAhBCAEhBCEEGotWzBQ+0ar24Wd5UgdWWCdUeCGlTlp8WqyEYDsKQggIIQghIIQghIAQghACjc1DP1MqdOQXfVJXdnnHiAMLfnR2AxsrIdiOghACQghCCAghCCEghCCEgBDCb7lNpU2N8tSW43XbTylv2LVjG/qRb1ZCsB0FIQSEEIQQEEIQQkAIQQiBxvYfjdZtJ6hxH42WWt5RNbBp2N5ZtUet0RPYjoIQmgIQQhBCQAhBCAEhBCEE2rsFXz/uA8ZSn3dVOCfbY0ttfhOZtKu2tzlyXFZCsB0FIQSEEIQQEEIQQkAIQQiBxm6m4K1xG9hEBhack+03v2qzoviEWwnBdhSEEBBCEEJACEEIASEEIQSEEH6LsrX3UuvOCovaIj2Lgj2mBj3XU6xFlbI1sB0FhBCEEBBCEEJACEEIASGEDkUrZrrt3hNUWLaSOuGFTY2ClSV5Iw82sIoPzEoItqMghIAQghACQghCCAghCCHQ2H7FzKCNQ1Kl1gkF6zMKa5giNTHlZStJx2UlBNtRQAhBCAEhBCEEhBCEEBBCEEJgab5qpyawEgJCCEIICCEIISCEIITAqiPtLf78/fP9/q8pg3PNj8XwOV2P1D3+y9qfP/uMb1/49l5wvRvBKfPTfpzPd+eN//561tzW36+E8/yn8+m43/99e3bFrOUhbARycQmtHezitt7/hdf0O+HGDewxWe5befeXzif29dpYjPl5r3T8ZuRyOvqd8LuZ2tg9nr4bWdsXHR/h34vj9V6zsQdb22KtXZdvb2SfTtRiqJ/O4Rc309dPPPi2R0Yoh/+vhAfvZJ+ettcz9/y/TvnE5zc8/p4bIzz4qucPWoxh8RX6+cb/es3tbtveHtRjF/fpHC4GfEpo13ZJ2yN8vGrc7xdZ29Gk7z8bq8opOfxiL/fdpuijwQd3mLuf9d0cBg95+0if173Fred1R3DiXfhS29HHmtBgUs79iMZnce2b8+vU9bnR+vrH7UWE1mbgyPs/59NvM8vvhG1+gTxxxte+wrUf/Otm7ALX1tvveLsH5Te8+u3o2+9LGzuTE2Pz0a7syAgPDv6jozjyl9e+c54yRadM+OJ9nke4Mfi84xrazj/Wx/Pw+pNj8MfMafNffo8P/sgIjw9+7dfR7fFvv2r719Evzlfwd+mPzuPGZ/nH+uXEam/xU7dki0/v3wm5PPHr0H8ksxbLZCVJugAAAABJRU5ErkJggg==) +![Answer](https://cdn.strawpoll.com/images/polls/qr/40Zm4E9AAga.png) -[Answer link](https://toreply.univ-lille.fr/reponse_4356) _Key: fw_ +[Answer link](https://strawpoll.com/40Zm4E9AAga) # BI vs Big Data @@ -728,6 +728,6 @@ What technologies are replacing Hadoop ecosystem (multiple choices)? - Answer C: Spark - Answer D: Cloud computing and object storage -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAG8klEQVR42u3d3a6iSBSAUZnm/d+4Q1+c5MQR5W+z2VWy1tWkp1Uo+QTtSjFM0/QA6oyPx2MYhu/bseCHy/KYtPzJFXk3Uwct+NJfeZT+7Ph/PoeglghBhCBCQIQgQkCEIEJAhHA/4+rfaHZ2SHAKReoMjMJpK5GHB8dk+aVTB7zro9SZEFyOgggBEYIIARGCCAERggiBi43Bx/c7DSIyvSN1ZklwwCP7FZzy0uwyMI0fpc6E4HIURAiIEEQIiBBECIgQRAhcbLztnnd6C6HUWUTBe1H1eysrZ0IQISBCECEgQhAhIEIQISBCECFwnftOW2t2taV+9xpnQhAhIEIQISBCECEgQhAhIELoQnTGTL+r9xSuSlQ4nyay1/2+141vuTMhiBBECIgQRAiIEEQIiBBECFxsfcaMlUVOF5m2Uri8TepLBye1dH2UOhOCCEGEgAhBhIAIQYSACEGEwMWGfhcO6XjQA9M7Vt+vwrVzCl/amRAQIYgQECGIEBAhiBAQIYgQECF0Y3w0vEhO4S3EUre82b0OzjtL3fK8VaTKF7ByJgSXoyBCQIQgQkCEIEJAhCBC4GLDNE2p6w6tvHzaS6fOLGl5zaJO383Ud6Rwss6WJ3cmBJejIEJAhCBCQIQgQkCEIELgYmPqsze7rMijdLGWyF5nPzyyU4X3Xcs7kC4YcGdCcDkKIgRECCIERAgiBEQIIgRECPeyvtBT4VykyEun3u8qqNn96nftrMJts9ATuBwFRAgiBEQIIgRECCIERAjdGZpdnCcodQZGcBWpZu+s1ulmPzqfCeRMCC5HQYSACEGEgAhBhIAIQYTAxZqeMVO4Fkvqkzc7kahwwwrVJmCNGXA5CiI0BCBCECEgQhAhIEIQIXC9oeWFQ1Y2veGbBHW6Xy2vMZM3LOUJOBOCy1EQISBCECEgQhAhIEIQISBCuJch9S5fhQs9NT3ogWFp+a5snU4kLF8TzJkQXI6CCAERgggBEYIIARGCCIGLjY/YRIdmb35WOxMosl+pL506WSf14XnKJyE5E4LLURAhIEIQISBCECEgQhAhcLFhmqa8CQH93r3snqvjNHtXti9mjRlwOQoiNAQgQhAhIEIQISBCECFwvTH4+MKJDqmTWoLTOwqXcsmb/5S6YUFd38DLmRBECCIERAgiBEQIIgRECCIERAj3Mva76YXTuwpvphWc/NXpDeFa3q/4e+1MCC5HQYSACEGEgAhBhIAIQYTAxaK3RktdbSl1uaTCJapavqlb3mZ/683P4u+mMyG4HAURAiIEEQIiBBECIgQRAhcbH6VLgxQ+eeSlU1cl6XfQmt3s4DIwkfdry6HiTAguR0GEgAhBhIAIQYSACEGEgAjhXtYXeupU7UJPhTc/u+f7VbgcmTMhuBwFRAgiBEQIIgRECCIERAjdGVf/RrMLBxVOgwi+dOot3yIv3fLNz/KOw/Ij3JkQXI6CCAERgggBEYIIARGCCIGLjcHH93uXr8jckcIbwqUOeHA+TWTLW15ZJ3uGkzMhuBwFEQIiBBECIgQRAiIEEQIXGw3BAatTKPIm3BQuiFK417X7lT1ozoTgchRECIgQRAiIEEQIiBBECIgQ7sW0tfeCSx41e/OzyEun7nWh8nfTmRBcjoIIARGCCAERgggBEYIIgYtFZ8w0u3pPyyKDFlxtqfBWdoWTkBofUmdCcDkKIgRECCIERAgiBEQIIgQutj5jptOFQ4IK551ENqxWZE5MyzOBgnN9nAnB5SggQhAhIEIQISBCECEgQhAh8D+DlZrAmRBECIgQRAiIEEQIiBBuZ8vyFn9+/3ua/hoyONfwczJ8ruunup8/+fTf+17j6APffhZ83wfBKeOTt2Fz802d74LP7t1nwmH40/hITdPfT8eEzPLGfEuW8z98+exu/+hq5Tvh2/F9HkcfaXnHer8Du1zgy2UUB78THjs+Fq4eT79QeXmDNz7n8xb+HjefLqt+/+TTxs8PsvnB9/aDbO9AvWzq3jGMfJguDM7Lpj6f/Xx8bz0Tnvtx9Tvu82vI5/91yis+P+H251zYwo2Pen6hl214OfiezwnzY/HT0bk8UD+H+N4xfNngswp0AXXa5WjS95+Ft+2UDg9cyy1s4d5HLfx0ETnyVl/r2Bju3eVjBWZ81N7lcvT3KuL6bxHtPNuxl3s7dG1+/J/74/bb8/P8E8RvMzu+E17zC+SJb8byL+NXbvz8m953HHafCnz7zVZmrVyOvv2+tPBbxYnZ7Loq27KFGzd+115s+cufvnOeMkQHvgPHDwB9Lo3z8j/Wx3uYX58Ef8x8LP6j8PaN37KF2zf+06+jy9u//KjlX0cPvF97f5fe+8z+sf5gLJa3uNWntfNS698J+Xrya9A/N7z+mLrBeZAAAAAASUVORK5CYII=) +![Answer](https://cdn.strawpoll.com/images/polls/qr/xVg71Demryr.png) -[Answer link](https://toreply.univ-lille.fr/reponse_748) _Key: qt_ +[Answer link](https://strawpoll.com/xVg71Demryr) diff --git a/src/03_Spark_Introduction.md b/src/03_Spark_Introduction.md index 0d0490b..17fd5a4 100644 --- a/src/03_Spark_Introduction.md +++ b/src/03_Spark_Introduction.md @@ -1,7 +1,7 @@ --- title: Spark Introduction author: Guillaume Eynard-Bontemps, CNES (Centre National d'Etudes Spatiales - French Space Agency) -date: 2020-11-15 +date: 2026 --- # Spark Introduction @@ -66,9 +66,9 @@ What are the main differences between Spark and Hadoop Map Reduce? - Answer C: Spark has a better name - Answer D: Spark provides many more APIs -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAEsCAIAAAD2HxkiAAAG5UlEQVR42u3d0W6jOhQF0OEq///LvQ+VUNQAMZhzjp2s9TSaNsE4bOzYxV5+fn7+AXX+UwVQ6/H2N5ZlWf+t2YTbLa/p+k3d7//s/fvcMa6+cPNe8Hk3glvqp6qEez9y7z7dHX2usjFN/UEuyzJ+Dbd0hfZ+9Od31mQ+3755/51ws6bWShz/bj2v9WKdMYGvSftzqcjhDd8Jr10fB73H2zsqfz7gxvd8LuF63bzea/78z17hXy+y14tv80Z2tqL+FPVsHZ69me5F6DVpv3fqkW8oo7eE996unu+R0R2V5zdsf8+DEja+avPe/1qM54ty8wLdu2SPK+r5cj9b/uR2SR/qRHc0qNIPWpVbcnjh1ntQwrOv2it8Z4Pw9ljX6vDsKUtgdnd07VEUfs0of7drh9usujGvvMxSSeCV74Sn+mYjXAd7X+HyC//6Te+3x/jBg0nP5/j2uzSp3dHN70sHYxU3xuZUr6ylhI2FP3UWLb98PN7YWUV3VfjByLkEnvtQ3n5z68zD65Bj52DmQdO3ObzZU8L2wu+Njh6X//hVx6OjFz6vC+PSjefV0hmRyd1KVjVfdefWRo3+nZCPJ35aQiB+YAYQQhBCQAhBCAEhBCEEhBCEEBBCEEJACEEIASEEIQSEEIQQEEIYSdIaMwPuB3Kw0VfV0Wtrvr1Ite9Ze4VoCUF3FBBCEEJACEEIgbtULoM/4DB9beEHLOfefow5L5/6CtESgu4oIIQghIAQghACx4bbqbdzsDhiULv9eYvO34x4sKNzkqDzPQecyBlwV1wtIQghCCEghCCEgBDCd3qogmdpsxGbIhZQatd5mhFzIVpCQAhBCAEhBCEEhBA+mCmK99IWJqp9iiJiesa8hZYQhBAQQhBCQAhBCIE9w01R1I5fpz0G0X7uERs/pE17zLILiJYQdEcBIQQhBIQQhBDIVzlFUbtHcUTh09aJGvA3XSFaQhBCQAhBCAEhBCEETkmaovjmtX3Szj1ijiFtkuCbrxAtIQghCCEghCCEgBDCdxpuoaeItZJql2+KKHzayyMqJOIjrj2QlhB0RwEhBCEEhBCEELhgyRmHrV1uKG3sPqJC2l/ebpYDRZQzYhJLSwi6o4AQghACQghCCFww3F4UnePCA05mpE26tFdyZ312fsQRNV971WkJQXcUEEIQQkAIQQiBCyr3ovi8BZQ6h78j5gPS3vPzFqTSEoLuKCCEIISAEIIQAqEqn6Ko3fygdhGhiApJe3ntuacVSUsIuqOAEIIQAkIIQgiEWmqXuNko0HgLE039V/+zTHukvbx21kRLCLqjgBCCEAJCCEIIrJKeohhwe+fauZkBN2loL+dmkTr34Yh4hiPt5VpC0B0FhBCEEBBCEELggomfohit5P8St2L+vCmftMW4BryWtISgOwpCCAghCCEghPCdvmKhp1n2oog4zYgDRdRSxBUy4CSWlhCEEBBCEEJACEEIgdVjilJOvYxPxIj8LOtERWwm0X5GnRtU2IsCdEcBIQQhBIQQhBAI9fiw86l9kqCznO0HSpuzqZ0LiTh67WyElhB0RwEhBCEEhBCEEFgtsyyGs1H08caa28vZrnY/hlm2iJhlakpLCLqjgBCCEAJCCEIIrCqnKKbeXDpi7L52k4ZZlkWq3VlESwi6o4AQghACQghCCNwlaaGntM0PBtwZu3NDhVqd8xadOqtulprXEoLuKAghIIQghIAQwneq3Isi7S/0Z/mr/7QnSGqXRYr4OGqPriUE3VFACEEIASEEIQQumGO77NpR/s4ipW0Z3S5tj4epz11LCLqjgBCCEAJCCEIIhHp8w0lGrALUORuRto90p9onXSIqxF4UgBCCEAJCCEIICCEMYhnwj8orqyNg/DpihiPt5QMO/dfWkpYQdEcBIQQhBIQQhBC4y6dtl92u8ymKtA0VIo6eNhvReYW0FyltKwstIeiOAkIIQggIIQghcJfhtsuOMOAiQmmnGbFjR8SWGwM+WuEpCtAdBYQQhBAQQhBCINRwe1HMMtLdWaSI7TE6zyht8aipNwDXEoLuKCCEIISAEIIQAnd5qIK3IqY9ardJiJgH6jzNiDmGiJrXEoLuKCCEIISAEIIQAncxRfFe2h7aA5aztvCdj5WkbWWhJQTdUUAIQQgBIQQhBC4YboqidrA4bVWlzqPXLt9UK+IpCi0h6I4CQghCCAghCCGQr3KKYsBdAdqlLWEUsarSgB9cxA7enZWsJQTdUUAIQQgBIQQhBEItA/5ROWgJASEEIQSEEIQQEEIQQkAIQQgBIQQhBIQQhBAQQhBCQAhBCAEhBCEEhBCEEBBCEEJACEEIASEEIQSEEIQQEEIQQkAIQQgBIYRp/A8pN6OQKpKSYgAAAABJRU5ErkJggg==) +![Answer](https://cdn.strawpoll.com/images/polls/qr/Dwyo3mBAeyA.png) -[Answer link](https://toreply.univ-lille.fr/reponse_102) _Key: rv_ +[Answer link](https://strawpoll.com/Dwyo3mBAeyA) ## APIs @@ -254,9 +254,9 @@ What's the main API of Spark? - Answer C: Datasets - Answer D: Transformations -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAEsCAIAAAD2HxkiAAAG+klEQVR42u3d0ZKjKhQF0Out/P8vZx66KpUySlA8HIhrPU1NdyISd0BoZHk+n/8Bef5XBZDr8fU3lmV5/VuzCZdbPtP1l7q//9n797FjnH3h5nfB730RXFI/0cXbrPzNH73/5zvf4F+6o3sVN46pP8JlWcav4fIXxPsXcflHzw9iVntPuHmVvOp38G/rqQ17pa4+9L3u0qqvxPX3hOeuj0Lv8fKbzNVnX/me7yV8XVKf3zWr/9kr/Of1V2g6WipqVdSjddj4ZZp1M3KLlvDab7L3jsrqPQvdm/Zj1b9noYQn+marMqxuod+bi89rce/qLFfUsiwn6nBV4KPd6c8vha9vKIHHuqOT9mFO9OUKJTz6qsIgRMuV9/VY5+rw6CmXA/9+q7L5hhJ4uDv61yZ06NZfe4jO9yF7d86fVTfmxXeiVO/RXUWu8G4SePKe8FDfrOd1UP9JR5d/r/Cfd3p77cPPKJygBCZ3Rwt3C5tjFRfG5lCvrKaElYU/dBY1v7x3z3lJFdUX9WsxNn8kgYc/lK93bo15+BxybBzMLDR9m8ObLSWsL/ze6Gi5/OVXlUdHT3xe58alz03WC+GBsKisW103EjL6PSE/T/y0hED8wAwghCCEgBCCEAJCCEIICCEIISCEIISAEIIQAkIIQggIIQghIIQwkk7PmBlwq5DCQ6Ozjp5b8+eev9b/PXOvEC0h6I4CQghCCAghCCFwlczH4A84TJ9b+AHLubcfY5+XT32FaAlBdxQQQhBCQAhBCIGy4XbqbRwsjhjUrl9v0fibEQs7GicJGt9zwImcAXfF1RKCEIIQAkIIQggIIdzTQxW86zYbsSl38L3xNCPmQrSEgBCCEAJCCEIICCH8MFMU3zU+mChiFUW3aQ/zFlpCEEJACEEIASEEIQSCDDdFkTt+HTGknvv0p4h1IbkTD783w6ElBCEEIQSEEIQQEEK4p8wpitw9iiMKn/ucqG5Hr//gcpeAaAkBIQQhBIQQhBAQQhhZpymKmzzbJ3dIvdvO2K4QLSEIISCEIISAEIIQAu2W0YaGI7aMjlgfUH/0brUUsTIj4jNqNMv241pC0B0FhBCEEBBCEEKgrNMqim6PMJplNmKWfaS71VK3R0JFXAxaQtAdBYQQhBAQQhBC4IROqygaR5Dr37Pb5tIRL6/XbcHELDUfUZ9aQtAdBYQQhBAQQhBCIFTmXhS5A9ARR+82dt/4m41n1Pib3Spklt22tYQghCCEgBCCEAJCCPf0SDy2nRtOHyhiPiBiLiTi3BsvsAHnLbSEIIQghIAQghACQgj39LjDSeYuLxjwr/4j5i3u/HItIeiOAkIIQggIIQghcELmdtmbGkfPZ3m2zyybNNTXfOMH1zgfkPtyLSHojgJCCEIICCEIIXDCkjs4u1Gg1EcDRRQ+dw1HY9V1+zgilkFEVIiWEHRHASEEIQSEEIQQuMrEUxSbuj3CqPHojXIPFFFLEVfIgJNYWkIQQkAIQQgBIQQhBF7m2Iti6sf4RIzIz/KcqIjdIOrPKHebdC0h6I4CQghCCAghCCFQlrmK4vfG2RvPfcDR86k/o1nWW2gJQXcUhBAQQhBCQAjhnjK3y+42Lpw7AF0/UD7LfgwDfnCN524VBeiOAkIIQggIIQgh0F+nVRS5f8+eu79FxLnn7g494MKOAffM0BKC7igghCCEgBCCEAJlmVMUA+5mPMvge+4W3LNs1t3t5VpC0B0FhBCEEBBCEELghOG2y576b+Qbjx7xqKVug++5u0HMMhuhJQTdUUAIQQgBIQQhBF4eU5Syflh5lscNNR5owMJHrLfIXZmhJQTdUUAIQQgBIQQhBEI97nCSjTMcuQ872pS7YfWAsxH2ogCEEIQQEEIQQkAIYTrLgH9Unlkd421tXX/0xsLP8oitiNOMKKeWEHRHASEEIQSEEIQQKOu0iqLb8oJ6uc+Jqi/SZtU11me32YhuReq2lYWWEHRHASEEIQSEEIQQuErmg55y942Y+kC59RnxkKsBl1ZYRQG6o4AQghACQghCCIQabi+KqUe6G3+z26YXEQfqtgQkopK1hKA7CgghCCEghCCEQH8PVfBVxJbREfMW9S//vQpprLrceQstIeiOghACQghCCAgh3JMpiu+6bb3Q7T27rTVpfM/6/UJyt7LQEoLuKCCEIISAEIIQAicMN0Uxy4bVmyKeUtX4rKSIxzflilhFoSUE3VFACEEIASEEIQT6y5yiGHBXgHq5z3TKfYRR7mYSjS8fcN5CSwi6oyCEgBCCEAJCCPe0DPhH5aAlBIQQhBAQQhBCQAhBCAEhBCEEhBCEEBBCEEJACEEIASEEIQSEEIQQEEIQQkAIQQgBIQQhBIQQhBAQQhBCQAhBCAEhBCEEhBCm8Q/Iybhp/ScYjAAAAABJRU5ErkJggg==) +![Answer](https://cdn.strawpoll.com/images/polls/qr/BDyNzM6WeyR.png) -[Answer link](https://toreply.univ-lille.fr/reponse_697) _Key: df_ +[Answer link](https://strawpoll.com/BDyNzM6WeyR) # Play with Map Reduce through Spark diff --git a/src/14_ObjectStorage.md b/src/14_ObjectStorage.md index 68c0a0c..9f73077 100644 --- a/src/14_ObjectStorage.md +++ b/src/14_ObjectStorage.md @@ -1,7 +1,7 @@ --- title: Object Storage and Cloud Optimized Datasets author: Guillaume Eynard-Bontemps, Hugues Larat, CNES (Centre National d'Etudes Spatiales - French Space Agency) -date: 2025 +date: 2026 --- # Object Storage @@ -13,8 +13,8 @@ date: 2025 But Why? -- Scalability. Scale-out, infinitly. -- Security/Reliability/Availability. Erasure Coding. +- Scalability. Scale-out. **Infinitly...** +- Security/Reliability/Availability. **Erasure Coding.** - Cost. Commodity hardware. - Performance (bandwith). Just need good network, and scale. @@ -40,9 +40,9 @@ What makes object storage efficient? (multiple choices) - Answer D: POSIX API - Answer E: Dedicated API developed for simple transactions -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAHFElEQVR42u3d3ZKiSBAGUNjm/d94gr1wo8dV/jRJsoBzYi8mulstSj6qdDOSfhzHDqgzdF3X9/31Diz14rI6Y8uvXjjhtdMSGdglz9LHgf/jOgS1hBCEEIQQEEIQQkAIQQgBIYT7GVb/otm6tmAJxfLDU4+6sJ4m9ckLj+vUZ6mVEGxHQQgBIQQhBIQQhBAQQhBC4GBD8PHnLYOIPHlwYIW9WCIjXx12s21gGj9LrYRgOwpCCAghCCEghCCEgBCCEAIHG2575M1WUTTbgaawWMdKCAghCCEghCCEgBCCEAJCCEIICCFcx33L1iLVYcEKrNQeU8sjjxxXs32crISAEIIQAkIIQggIIQghIIQghMBHohUz5+3eU9gxKXVK896R877XjY/cSghCCEIICCEIISCEIISAEIIQAgdbr5i5Z2eRYLeVvGYtqS9deNTBopZTn6VWQhBCEEJACEEIASEEIQSEEIQQOFh/3sYhufOSWYERmfPzlq0406yEIISAEIIQAkIIQggIIQghIIQghEDXPcrWmm2Sk9cuKXVg3WkLuFruxXThBlZWQrAdBSEEhBCEEBBCEEJACEEIgYP1weKPVKmVJc3WWFx1SguPq7BYZ8uTWwnBdhSEEBBCEEJACEEIASEEIQQONnSZnUUKK0tWh13Y6KXZeprUdzP71asmPH7UVkKwHQUhBIQQhBAQQhBCQAhBCAEhhHsZVv8itXYsorAWqfAmYS035iosMzzpUXcaPYHtKCCEIIQghIAQghACQghCCFQYgo8vbJeU15+qK23+k1oa0mz9U6rglGZPmpUQbEdBCAEhBCEEhBCEEBBCEELgYP04jnn9OZqtp2l55JVnQ6t3ZUtVe0c3PWbAdhSE0BSAEIIQAkIIQggIIQghcLziHjORPjGpNTGFtSOFU5o6sKCTVvPoMQO2o4AQghACQghCCAghCCEghCCEwKuha7iAa1lhUduqvNqxLY2D2jzq1Lc7ddjZ75eVEGxHQQgBIQQhBIQQhBAQQhBC4GDrjZ4ihSmFN51afemrNg5q9kZ3zd5tLjil8eOyEoLtKAghIIQghIAQghACQghCCBxs6EqrXvLKcYKNQ1Ln5KTFOqnDPumc7DJyKyHYjoIQAkIIQggIIQghIIQghMDB+nEcCytmVgYXqEVotqNJ6lF3mbcQuuc9m6yEYDsKCCEIISCEIISAEIIQAkIIQgjsbAg+vtkmUS2PvNlyvNTeWc2+m+U9wayEYDsKQggIIQghIIQghIAQghACBytu9JTX/KfleprU8o4897whXPa0WAnBdhSE0BSAEIIQAkIIQggIIQghcLz+1PeUCh35Re+7dtIb3RUWZq2+dHYDGysh2I6CEAJCCEIICCEIISCEIISAEMK9DN1FO/AEy6AKmxo1W1lWK6/qrfyWb1ZCsB0FIQSEEIQQEEIQQkAIQQiBgw2rf9FsU6NgeUekP09hOU5qUUvLNz/LOw9rj1qjJ7AdBSE0BSCEIISAEIIQAkIIQggcbwg+/qRFEt1pO+sE7+OV2vwmUoR01fY2W47LSgi2oyCEgBCCEAJCCEIICCEIIXCwwRRMSu0iU1gdUnj3oqs2K4pPuJUQbEdBCAEhBCEEhBCEEBBCEEJACOFelK1NK7w1WmoVVeS4gj2mmpXaO0vZGtiOAkIIQggIIQghIIQghIAQQoOiFTPNdu9JFWx5lHp/suDI806GYGVJ3sgL300rIdiOAkIIQghCCAghCCEghCCEQIn1ipmTNg45r2B9RmENU6QmprxsJem4rIRgOwoIIQghIIQghIAQghACQghCCLzq79mpCayEgBCCEAJCCEIIQmgKQAjh1ra0t/j5/fc4/jFlsK/+sRg+p+uRusdP5v792Wt8+8DJa8H1LgS7zE/ewN79DnXhAj33q5fndFn/ux2dm+52nPrd6vuf9md4cs5f/pu8djxfo5d/9fzz90f5TDg9Hb8z1ezV+gLez+/Trd7PiVr41cuRyuHWz4TfnR8Lu8fdP2R+t8N5HuHvKfJ+rXn5ycZd1uQZNnkh+3SiXob66RxGLqbPj114hu1P7sr+v5Vw32vS82Xv5TkX9jDx19r+nAsj3PiohV3W3LV/csWbOwWXJ6rvf76Yw5cB7/jBdeG3c5ceCZzYjqZuVFb3MEfu5RZG+N0GbPcd5uprfTeHnx5yUgL3vQpfajv6WBMOmJR9X+Lgd3Huk/P71LV5jd/9y+0vEvg+bzdfD4f3SWnwPNh+fmSPf27wk9utC5xbeQkkcTs6+Xlp4buKHWPz0a5sywg3Dv6jo9jyx3OfOXeZoi8+A++VwJdXF9T/5mH5f9bH8/D+lWPwy8yFpW/y683ICLcPfu7b0eXxLz9q+dvRL96vT7+XXo3Z3A5zy5Wl5U370SHU3uJWl2SLT+ufCbk88WvQv3r0K3SEaLMqAAAAAElFTkSuQmCC) +![Answer](https://cdn.strawpoll.com/images/polls/qr/6QnMQkd2bne.png) -[Answer link](https://toreply.univ-lille.fr/reponse_7223) _Key: sl_ +[Answer link](https://strawpoll.com/6QnMQkd2bne) # Processing and Cloud Optimized datasets @@ -186,6 +186,6 @@ What is Cloud Optimized? Object storage is always performant for scientific data processing. -![Answer](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAASwAAAFACAIAAACJKdYDAAAG+klEQVR42u3d27LaOBAFUDzx///xlOeBqhQD+ALtllpmradUcjjIsrclSFd7WpblBvQz3263aZqud2DBm8v2nFS+c0XOZuqkBd/6klfp/cD/cR+CvoQQhBCEEBBCEEJACEEIASGE3zPv/kTZ6pBgCUVqBUbHspXIy4Nzsv3WqRM+9FVqJQTbURBCQAhBCAEhBCEEhBCEEGhsDr5+3DKISHlHamVJcMIjxxUseSnbBqb4VWolBNtREEJACEEIASEEIQSEEIQQaGz+2SMf9BFCqVVEwWdRjfsoKyshCCEghCCEgBCCEAJCCEIICCEIIdDO75atle22NO5RYyUEIQSEEIQQEEIQQkAIQQgBIYQhRCtmxu3e07ErUcd6mshRj3uui4/cSghCCEIICCEIISCEIISAEIIQAo3tV8zoLHK6SNlKx/Y2qW8dLGoZ+iq1EoIQghACQghCCAghCCEghCCEQGPTuI1DBp70QHnH7vnq2Dun41tbCQEhBCEEhBCEEBBCEEJACEEIASGEYcy3wk1yOj5CLHXkZY86WHeWOvK8LlLdG1hZCcF2FIQQEEIQQkAIQQgBIQQhBBqblmXJ68/TsRah8nO8cs9oZhepvLcOKlusc+SXWwnBdhSEEBBCEEJACEEIASEEIQQam2/DPrOqclFLx7KVvAMPFiGVndLUehorIdiOAkIIQggIIQghIIQghIAQghACz+bdnxj0wWmpx9Wx0KlyA6vUYr2yz8nT6AlsRwEhBCEEhBCEEBBCEEJACGE4+49Gi9gtgyhbjpN6XGU7a131gXAdq22OTJqVEGxHQQgBIQQhBIQQhBAQQhBCoLE5+PrUKorUXiwRqY1eUpWd0lQdK4GOXCpWQrAdBSEEhBCEEBBCEEJACEEIgcaiFTOVaxG2deyY0rGeJvXBSamu2o7ISgi2oyCEpgCEEIQQEEIQQkAIQQgBIYSfM/V9elleJVRqUVvHRk+Vn8p21UsltTuWRk9gOwpCaApACEEIASEEIQSEEIQQaG9O/e3BKoeOXYnKPr0sOLCOU9q3T1RkYNlFSFZCsB0FIQSEEIQQEEIQQkAIQQiBxqZlWSIFAR1buXSswEjtStLzaujXJGbQJ5+dMi1WQrAdBSEEhBCEEBBCEEJACEEIgcaiPWbGradJLccp2x0n8ssrt+0ZtETJSghCCAghCCEIISCEIISAEIIQAkIIvyhattax+Cu1UqlsTVxqj6lgEWLZmrjU44qfaysh2I6CEAJCCEIICCEIISCEIIRAY9FHo+389n7Pu+pbrFO21qfslI4rfjathGA7CkIICCEIISCEIISAEIIQAo3Nt8wCjsoPrCrbleTCc95r2Kn1T8HXLstiJQTbURBCQAhBCAEhBCEEhBCEEBBC+C37jZ4GlVqLNGinplvXerrU85V6XNmVgFZCsB0FIQSEEIQQEEIQQkAIQQiBxubdnyjbOCi1DCK1dqRjsU7qM9sGbWDV/Qq3EoLtKAghIIQghIAQghACQghCCDQ2B18/7lO+sp93lXTUHUt5dg8qMvLKjY6yK5yshGA7CkIICCEIISCEIISAEIIQAo3NpuALwVKevHqaVME6oas2K4pPmpUQbEdBCAEhBCEEhBCEEBBCEEJACOG3KFtLUfbhZ5G33h1Y5WZNvY5LoyewHQWEEIQQEEIQQkAIQQgBIYSCohUzZbv3pB5XsDQkMmnBbksdH2UXfO5a3si7T6mVEGxHQQgBIQQhBIQQhBAQQhBCoLH9iplBG4dcVeUSpUhNTOVKoGCtj5UQbEcBIQQhBIQQhBAQQhBCQAhBCIH/ma7aqQmshIAQghACQghCCAghCCGw6kh7iz9//7ws/5oyONd0Xwwf03VP3f1v1v782Xt8+8K394Lr3QhOmZ/s4b2d/O0bdPHjKrcdfZzNmoY+l9P0p/4Mb98gHm/Eu/80xBVV8TPh21n7O7/uaqn3l5oT+3TS17ZLT3slCTz/M+F318fG7vH0D5lPp/zg73wc4d9L6vVe8/Q3a4N/vew2lo7IRD0N9dM5DN5M3wZy494tYB+shOfO2uNG5e0uZW0PE3yv479zY4Rf7M2exvD0Efrxwn29dteu5t3N3hdz+PVKdd8KbbzKRumc7Wj3PUyzvdzGCD991drggzvM3ff6bg4/PeSDgZfA07aj9zWhwRbi3LdovOdZ++T8OnU1L8ovRvUY3fthbtxYiX4m/Ghv1vI6OL4Lyh7/2uBfP+k9XayXJIFFt6MbX5S9/a7ixNh8tCs7MsKDg//oKI788NpnzlOm6PhQt4chgaddt9v/WR/Pw+tXjsEvMzeWvrdfb0ZGeHzwa9+Obo9/+1Xb345+cb6++1764Mi3P6PK6tYMa2/xUzd1y1f1z4RcnvgV9B8NeOCjymp7zgAAAABJRU5ErkJggg==) +![Answer](https://cdn.strawpoll.com/images/polls/qr/kjn1DxzMjyQ.png) -[Answer link](https://toreply.univ-lille.fr/reponse_621) _Key: kj_ +[Answer link](https://strawpoll.com/kjn1DxzMjyQ)