Docker Project 3-DockerCompose, Elasticsearch

Configure Docker Compose to run Python container.

 

1. Create Dockerfile to run Jupyternotebook.

FROM continuumio/miniconda3
  
WORKDIR /usr/app

RUN mkdir result

RUN mkdir src

EXPOSE 8888

RUN pip install --no-cache-dir --upgrade pip

COPY ./requirements.txt .

RUN pip install -r requirements.txt

RUN pip install jupyter

RUN jupyter notebook --generate-config --allow-root

ENTRYPOINT jupyter notebook --allow-root --ip=0.0.0.0 --port=8888 --no-browser

 

2. Define services in a Compose file.

version: '3.0'

services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es01
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - cluster.initial_master_nodes=es01
      - bootstrap.memory_lock=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data01:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
    networks:
      - elastic

  jupyter:
    build:
      context: . #A build's context is the set of files located in the specified PATH or URL.
    ports:
      - 8888:8888
    volumes:
      - ./result:/usr/app/result #File structure refered by the screen shot below.
      - ./src:/usr/app/src
    networks: #Node es01 listens on localhost:9200 and jupyter talks to es01 over a Docker network. 
      - elastic

volumes:
  data01: #It stores the node data directory so the data persists across starts.
    driver: local

networks:
  elastic:
    driver: bridge

file structure

 

3. Build and run with Compose.

$ docker-compose up
Recreating workspace_jupyter_1 ... done
Starting es01                  ... done

...

 

4. Configure environment values to access local Elasticsearch.

- Create .env file (You can set default values for environment variables using a .env, which Compose automatically looks for in project directory (parent folder of your Compose file)).

Workspace joohyunyoon$ vi .env

$ vi .env

ELASTIC_HOST=###.###.##.###
ELASTIC_PORT=####
ELASTIC_USER=
ELASTIC_KEY=

 

- Set environment variables in docker-compose.yml

version: '3.0'

services:
  jupyter:
    build:
      context: .
    ports:
      - 8888:8888
    volumes:
      - ./result:/usr/app/result
      - ./src:/usr/app/src
    env_file: #<<<<<<<<<<<
      - ./.env

  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2
    container_name: es01

 

5. Run using with environment file.

$ docker-compose --env-file ./.env up #<<<<<<<<
Starting es01                  ... done
Recreating workspace_jupyter_1 ... done

...

Jupyternotebook Result

 

6. Input the data into Elasticsearch

-Setup in docker-compose.yml.

version: '3.0'

services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es01
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - cluster.initial_master_nodes=es01
      - bootstrap.memory_lock=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data01:/usr/share/elasticsearch/data   
    ports:
      - 9200:9200 
    networks:
      - elastic

  jupyter:
    build:
      context: . #The build process can refer to any of the files in the context.
    ports:
      - 8888:8888
    volumes:
      - ./result:/usr/app/result
      - ./src:/usr/app/src
    env_file:
      - ./.env
    networks:
      - elastic
    environment:
      - ELASTIC_HOST=es01 #<<<<I am going to use same network as es01 to interact with es01
      - ELASTIC_PORT=9200

volumes:
  data01:
    driver: local

networks:
  elastic:
    driver: bridge

 

- Elasticsearch setup in code.

from elasticsearch.connection import create_ssl_context
from elasticsearch import Elasticsearch, helpers
from datetime import datetime
import os

ELASTIC_HOST=os.getenv('ELASTIC_HOST')
if ELASTIC_HOST == None:
    ELASTIC_HOST='http://localhost'
ELASTIC_PORT=os.getenv('ELASTIC_PORT')
if ELASTIC_PORT == None:
    ELASTIC_PORT=9200
else:
    ELASTIC_PORT=int(ELASTIC_PORT)
    
es=Elasticsearch(hosts=[{'host': ELASTIC_HOST, 'port': ELASTIC_PORT}])

 

- Input the data

<First way> 

def insertData():
    index="product_list"
    doc={
        "category":"skirt",
        "c_key":"1234",
        "price":11400,
        "status":1,
        "@timestamp":datetime.utcnow().strftime('%Y-%m-%dT%H:%S.%f')[:-3]+'Z'
    }
    res=es.index(index="product_list",doc_type="_doc",body=doc)
    print(res)
insertData()

>>>

{'_index': 'product_list', '_type': '_doc', '_id': 'nn3VJn0BmVRjSit0F-s5', '_version': 1, 'result': 'created', '_shards': {'total': 2, 'successful': 1, 'failed': 0}, '_seq_no': 124, '_primary_term': 4}

<Second way>

def searchAPI():
    index="product_list"
    body={
        "query":{
            "match_all":{}
        }
    }
    res=es.search(index=index, body=body)
    # data = res['hits']['hits']
    print(res)
searchAPI()

>>>

{'took': 5, 'timed_out': False, '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0}, 'hits': {'total': {'value': 124, 'relation': 'eq'}, 'max_score': 1.0, 'hits': [{'_index': 'product_list', '_type': '_doc', '_id': 'tyIhI30BsUxRYd-gkWUc', '_score': 1.0, '_source': {'category': 'skirt', 'c_key': '1234', 'price': 11400, 'status': 1, 

...

<Third way>

docs=[]
for num in range(3):
    docs.append({
        '_index':"product_list",
        '_source':{
            "category":"skirt",
            "c_key":"1234",
            "price":11400,
            "status":1,
            "@timestamp":datetime.utcnow().strftime('%Y-%m-%dT%H:%S.%f')[:-3]+'Z'
        }
    })
print(docs)
helpers.bulk(es,docs)

>>>

[{'_index': 'product_list', '_source': {'category': 'skirt', 'c_key': '1234', 'price': 11400, 'status': 1, '@timestamp': '2021-11-16T03:33.145Z'}}, {'_index': 'product_list', '_source': {'category': 'skirt', 'c_key': '1234', 'price': 

...