ExperimentsWithDocker/docker-compose_spark_notebook.yml at master · miciav/ExperimentsWithDocker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
version: '2'

networks:
  prodnetwork:
    driver: bridge

services:
  zookeeper:
    image: wurstmeister/zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    networks:
      - prodnetwork
  kafka:
    image: wurstmeister/kafka
    container_name: kafka
    ports:
      - "9092:9092"
    environment:
#      KAFKA_ADVERTISED_HOST_NAME: HOSTIP
#      KAFKA_CREATE_TOPICS: "topic:1:1"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
#      KAFKA_ADVERTISED_HOST_NAME: kafka
      KAFKA_ADVERTISED_PORT: 9092
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - prodnetwork
    depends_on:
      - zookeeper
  producer:
    container_name: producer
    image: miciav/producer
    environment:
       KAFKA_ADDRESS: kafka:9092
       ZK_ADDRESS: zookeeper:2181
    depends_on:
      - kafka
      - zookeeper
    networks:
      - prodnetwork
    links:
      - kafka
    restart: on-failure:10
  producer:
    container_name: producer
    image: miciav/producer
    environment:
       KAFKA_ADDRESS: kafka:9092
       ZK_ADDRESS: zookeeper:2181
    depends_on:
      - kafka
      - zookeeper
    networks:
      - prodnetwork
    restart: on-failure:10
  master:
    image: gettyimages/spark
    command: bin/spark-class org.apache.spark.deploy.master.Master -h master
    container_name: master
    environment:
      MASTER: spark://master:7077
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: localhost
    expose:
      - 7001
      - 7002
      - 7003
      - 7004
      - 7005
      - 7006
      - 7077
      - 6066
    ports:
      - 4040:4040
      - 6066:6066
      - 7077:7077
      - 9000:8080
    volumes:
      - ./conf/master:/conf
      - ./data:/tmp/data
    networks:
      - prodnetwork
  worker:
    image: gettyimages/spark
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
    container_name: worker
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 1g
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8081
      SPARK_PUBLIC_DNS: localhost
    links:
      - master
    expose:
      - 7012
      - 7013
      - 7014
      - 7015
      - 7016
      - 8881
    ports:
      - 8081:8081
    volumes:
      - ./conf/worker:/conf
      - ./data:/tmp/data
    networks:
      - prodnetwork
  # zeppelin:
  #   container_name: zeppelin
  #   image: dylanmei/zeppelin
  #   environment:
  #     ZEPPELIN_PORT: 8080
  #     ZEPPELIN_JAVA_OPTS: >-
  #       -Dspark.driver.memory=0.1g
  #       -Dspark.executor.memory=0.1g
  #     MASTER: local[*]
  #     KAFKA_ADDRESS: kafka:9092
  #   depends_on:
  #     - kafka
  #     - zookeeper
  #     - cassandra
  #   links:
  #     - cassandra
  #     - master
  #   ports:
  #     - "8080:8080"
  #   volumes:
  #     - ./zeppelin/about.json:/usr/zeppelin/notebook/about.json
  #     - ./zeppelin/data:/usr/zeppelin/data
  #     - ./zeppelin/notebooks:/usr/zeppelin/notebook
  #     - ./zeppelin/conf:/usr/zeppelin/conf
  #   networks:
  #     - prodnetwork
  cassandra:
    container_name: cassandra
    image: spotify/cassandra
    ports:
      - "7001:7001"
      - "7199:7199"
      - "9160:9160"
      - "7000:7000"
      - "9042:9042"
    volumes:
      - ./cassandra:/var/lib/cassandra
    networks:
      - prodnetwork
  spark-notebook:
    image: earthquakesan/hadoop-spark-notebook:1.0.0
    hostname: sparknotebook
    container_name: sparknotebook
    depends_on:
      - kafka
      - zookeeper
      - cassandra
    links:
      - cassandra
      - master
    networks:
      - prodnetwork
    environment:
      - SPARK_NOTEBOOK_MASTER=spark://master:7077
#    env_file:
#      - ./hadoop.env
    ports:
      - "9100:9000"