chore: bump deps, refactor Py tests to use dict options (#25)

* chore: bump deps

* ci: bump pip

* ci: remove 3.8 Python

* Update README.md

Author: Anush008

.github/workflows/test.yml

@@ -14,7 +14,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - '3.8'
           - '3.9'
           - '3.10'
           - '3.11'
@@ -42,6 +41,8 @@ jobs:
           python-version: '${{ matrix.python-version }}'
           cache: pip
       - name: Install Python test dependencies
-        run: pip install -r src/test/python/requirements.txt
+        run: |
+          pip install --upgrade pip
+          pip install -r src/test/python/requirements.txt
       - name: Run Python tests
         run: pytest

.releaserc

@@ -68,73 +68,6 @@
                 ],
                 "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
             }
-        ],
-        [
-            "@semantic-release/release-notes-generator",
-            {
-                "preset": "conventionalcommits",
-                "parserOpts": {
-                    "noteKeywords": [
-                        "BREAKING CHANGE",
-                        "BREAKING CHANGES",
-                        "BREAKING"
-                    ]
-                },
-                "writerOpts": {
-                    "commitsSort": [
-                        "subject",
-                        "scope"
-                    ]
-                },
-                "presetConfig": {
-                    "types": [
-                        {
-                            "type": "feat",
-                            "section": "🌞 Features"
-                        },
-                        {
-                            "type": "fix",
-                            "section": "🐛 Bug Fixes"
-                        },
-                        {
-                            "type": "perf",
-                            "section": "🚀 Performance Improvements"
-                        },
-                        {
-                            "type": "revert",
-                            "section": "⏩ Reverts"
-                        },
-                        {
-                            "type": "docs",
-                            "section": "📝 Documentation"
-                        },
-                        {
-                            "type": "style",
-                            "section": "🎨 Styles"
-                        },
-                        {
-                            "type": "refactor",
-                            "section": "🧑‍💻 Code Refactoring"
-                        },
-                        {
-                            "type": "test",
-                            "section": "✅ Tests"
-                        },
-                        {
-                            "type": "build",
-                            "section": "🤖 Build System"
-                        },
-                        {
-                            "type": "ci",
-                            "section": "🔁 Continuous Integration"
-                        },
-                        {
-                            "type": "chore",
-                            "section": "🧹 Chores"
-                        }
-                    ]
-                }
-            }
         ]
     ]
 }

README.md

@@ -192,7 +192,8 @@ You can use the connector as a library in Databricks to ingest data into Qdrant.
 - Select `Install New` to open the library installation modal.
 - Search for `io.qdrant:spark:VERSION` in the Maven packages and click `Install`.
 
-<img width="1064" alt="Screenshot 2024-01-05 at 17 20 01 (1)" src="https://github.com/qdrant/qdrant-spark/assets/46051506/d95773e0-c5c6-4ff2-bf50-8055bb08fd1b">
+<img width="704" alt="Screenshot 2024-04-28 at 11 34 17 AM" src="https://github.com/qdrant/qdrant-spark/assets/46051506/0c1bd356-3fba-436a-90ce-d8ff39b02d1f">
+
 
 ## Datatype support
 

pom.xml

@@ -39,12 +39,12 @@
 		<dependency>
 			<groupId>io.qdrant</groupId>
 			<artifactId>client</artifactId>
-			<version>1.8.0</version>
+			<version>1.9.0</version>
 		</dependency>
 		<dependency>
 			<groupId>com.google.guava</groupId>
 			<artifactId>guava</artifactId>
-			<version>32.0.0-jre</version>
+			<version>33.1.0-jre</version>
 		</dependency>
 		<dependency>
 			<groupId>io.grpc</groupId>
@@ -56,7 +56,7 @@
 		<dependency>
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
-			<version>2.0.7</version>
+			<version>2.0.13</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.spark</groupId>
@@ -75,13 +75,13 @@
 		<dependency>
 			<groupId>org.testcontainers</groupId>
 			<artifactId>qdrant</artifactId>
-			<version>1.19.6</version>
+			<version>1.19.7</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.testcontainers</groupId>
 			<artifactId>junit-jupiter</artifactId>
-			<version>1.19.4</version>
+			<version>1.19.7</version>
 			<scope>test</scope>
 		</dependency>
 	</dependencies>

src/test/python/conftest.py

@@ -1,24 +1,26 @@
 import pytest
-from testcontainers.core.container import DockerContainer  # type: ignore
-from testcontainers.core.waiting_utils import wait_for_logs  # type: ignore
+from testcontainers.qdrant import QdrantContainer
 from qdrant_client import QdrantClient, models
 import uuid
 from pyspark.sql import SparkSession
 from typing import NamedTuple
+from uuid import uuid4
 
 
 QDRANT_GRPC_PORT = 6334
 QDRANT_EMBEDDING_DIM = 6
 QDRANT_DISTANCE = models.Distance.COSINE
+QDRANT_API_KEY = uuid4().hex
 
 
 class Qdrant(NamedTuple):
     url: str
+    api_key: str
     collection_name: str
     client: QdrantClient
 
 
-qdrant_container = DockerContainer("qdrant/qdrant").with_exposed_ports(QDRANT_GRPC_PORT)
+qdrant_container = QdrantContainer(image="qdrant/qdrant:latest", api_key=QDRANT_API_KEY)
 
 
 # Reference: https://gist.github.com/dizzythinks/f3bb37fd8ab1484bfec79d39ad8a92d3
@@ -36,9 +38,6 @@ def get_pom_version():
 @pytest.fixture(scope="module", autouse=True)
 def setup_container(request):
     qdrant_container.start()
-    wait_for_logs(
-        qdrant_container, ".*Actix runtime found; starting in Actix runtime.*", 60
-    )
 
     def remove_container():
         qdrant_container.stop()
@@ -70,6 +69,8 @@ def qdrant():
         host=host,
         grpc_port=grpc_port,
         prefer_grpc=True,
+        api_key=QDRANT_API_KEY,
+        https=False,
     )
 
     collection_name = str(uuid.uuid4())
@@ -99,6 +100,7 @@ def qdrant():
         url=f"http://{host}:{grpc_port}",
         client=client,
         collection_name=collection_name,
+        api_key=QDRANT_API_KEY,
     )
 
     return client.close()

src/test/python/requirements.txt

@@ -1,4 +1,4 @@
 pyspark==3.5.1
-pytest==8.0.2
-qdrant-client==1.7.3
-testcontainers==3.7.1
+pytest==8.2.0
+qdrant-client==1.9.0
+testcontainers==4.4.0