Skip to content

Commit

Permalink
chore: update meilisearch scraper and client (#1025)
Browse files Browse the repository at this point in the history
  • Loading branch information
diogotcorreia authored Nov 7, 2024
1 parent a12073f commit 9744fb1
Show file tree
Hide file tree
Showing 10 changed files with 67 additions and 20 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/docs-scraper.config.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"index_uid": "resumos-leic-v2",
"index_uid": "resumos-leic",
"start_urls": ["https://resumos.leic.pt/"],
"selectors": {
"lvl0": {
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/gh-page-scraping.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
scrape-site-for-search:
name: 'Scrape Site for Search'
if: github.event.deployment_status.state == 'success' && github.event.deployment.environment == 'production'
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Run docs-scraper
Expand All @@ -20,4 +20,4 @@ jobs:
-e MEILISEARCH_HOST_URL=$HOST_URL \
-e MEILISEARCH_API_KEY=$API_KEY \
-v $CONFIG_FILE_PATH:/docs-scraper/config.json \
getmeili/docs-scraper:v0.12.1 pipenv run ./docs_scraper config.json
getmeili/docs-scraper:v0.12.12 pipenv run ./docs_scraper config.json
15 changes: 15 additions & 0 deletions .meilisearch/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Meilisearch

## Commands to generate keys

```sh
curl https://meilisearch.diogotc.com/keys \
-H "Authorization: Bearer $MEILI_TOKEN" \
--json '{"actions": ["search"], "indexes": ["resumos-leic"], "expiresAt": null, "description": "Search key for Resumos LEIC"}'
```

```sh
curl https://meilisearch.diogotc.com/keys \
-H "Authorization: Bearer $MEILI_TOKEN" \
--json '{"actions": ["documents.*", "indexes.*", "settings.*"], "indexes": ["resumos-leic"], "expiresAt": null, "description": "Scraping key for Resumos LEIC"}'
```
2 changes: 1 addition & 1 deletion .meilisearch/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ version: '3.8'

services:
meilisearch:
image: getmeili/meilisearch:v0.20.0
image: getmeili/meilisearch:v1.7.6
ports:
- 7700:7700
volumes:
Expand Down
5 changes: 5 additions & 0 deletions gatsby-config.js
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ module.exports = {
contributorsLink: 'https://github.com/leic-pt/resumos-leic/graphs/contributors',
vercelLink: 'https://vercel.com/?utm_source=leic-pt&utm_campaign=oss',
},
search: {
host: 'https://meilisearch.diogotc.com',
apiKey: 'a66ec2f3c48d2f827a81e850de53d2a764b5d5f420111e15c686eec6885480f5',
indexName: 'resumos-leic',
},
},
plugins: [
{
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
"gatsby-source-filesystem": "^5.13.0",
"gatsby-transformer-remark": "^6.13.0",
"katex": "^0.16.9",
"meilisearch": "^0.30.0",
"meilisearch": "^0.41.0",
"mermaid": "^10.9.0",
"mini-svg-data-uri": "^1.4.4",
"prismjs": "^1.29.0",
Expand Down
32 changes: 26 additions & 6 deletions src/components/SearchBar/SearchBar.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
import MeiliSearch from 'meilisearch';
import React, { useCallback, useEffect, useState } from 'react';
import { useStaticQuery, graphql } from 'gatsby';
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import Dialog from '../Dialog/Dialog';
import Search from '../icons/Search';
import './SearchBar.css';
import SearchModal from './SearchModal';

const searchClient = new MeiliSearch({
host: 'https://meilisearch.diogotc.com',
apiKey: 'S3goii63d54d41ee506eb2bdfea46f62cb3b90a3141ec34ee3e546db99dbffd73a7872e9',
});

const SearchBar = () => {
const [open, setOpen] = useState(false);
const [filterBySection, setFilterBySection] = useState(true);
Expand All @@ -22,6 +18,29 @@ const SearchBar = () => {
[setFilterBySection]
);

const data = useStaticQuery(graphql`
query SearchConfigQuery {
site {
siteMetadata {
search {
host
apiKey
indexName
}
}
}
}
`);
const { host, apiKey, indexName } = data.site.siteMetadata.search;
const searchClient = useMemo(
() =>
new MeiliSearch({
host,
apiKey,
}),
[host, apiKey]
);

// Global keybinds
useEffect(() => {
const handleKeyPress = (event) => {
Expand Down Expand Up @@ -75,6 +94,7 @@ const SearchBar = () => {
<Dialog open={open} onClose={handleCloseSearch}>
<SearchModal
searchClient={searchClient}
indexName={indexName}
onClose={handleCloseSearch}
filterBySection={filterBySection}
handleToggleFilterBySection={handleToggleFilterBySection}
Expand Down
9 changes: 8 additions & 1 deletion src/components/SearchBar/SearchModal.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,13 @@ const initialState = {
status: 'idle',
};

const SearchModal = ({ searchClient, onClose, filterBySection, handleToggleFilterBySection }) => {
const SearchModal = ({
searchClient,
indexName,
onClose,
filterBySection,
handleToggleFilterBySection,
}) => {
// Refs to elements of search, to use with autocomplete-core
const formElementRef = React.useRef(null);
const inputRef = React.useRef(null);
Expand Down Expand Up @@ -44,6 +50,7 @@ const SearchModal = ({ searchClient, onClose, filterBySection, handleToggleFilte
},
getSources: createGetSources({
searchClient,
indexName,
onClose,
section: filterBySection && currentSection,
}),
Expand Down
4 changes: 2 additions & 2 deletions src/components/SearchBar/autocomplete.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import { navigate } from 'gatsby';

export function createGetSources({ searchClient, onClose, section }) {
export function createGetSources({ searchClient, indexName, onClose, section }) {
return async ({ query, setContext, setStatus }) => {
if (!query) {
// Return no results if query is empty
return [];
}

try {
const { hits, nbHits } = await searchClient.index('resumos-leic-v2').search(query, {
const { hits, nbHits } = await searchClient.index(indexName).search(query, {
attributesToHighlight: [
'hierarchy_lvl1',
'hierarchy_lvl2',
Expand Down
12 changes: 6 additions & 6 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -4080,7 +4080,7 @@ create-gatsby@^3.13.1:
dependencies:
"@babel/runtime" "^7.20.13"

cross-fetch@^3.1.5:
cross-fetch@^3.1.5, cross-fetch@^3.1.6:
version "3.1.8"
resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82"
integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==
Expand Down Expand Up @@ -8313,12 +8313,12 @@ [email protected]:
resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748"
integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==

meilisearch@^0.30.0:
version "0.30.0"
resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.30.0.tgz#707f9a6b07440c496b965379616e084f112160ae"
integrity sha512-3y1hALOwTDpquYar+gDREqRasFPWKxkWAhk6h+RF+nKObPVf9N77wcTNvukGwOKbxRyJnKge0OPgAB1BkB9VVw==
meilisearch@^0.41.0:
version "0.41.0"
resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.41.0.tgz#98fc50dd8ce4f2446aa4445e3cecd47e3c9afde5"
integrity sha512-5KcGLxEXD7E+uNO7R68rCbGSHgCqeM3Q3RFFLSsN7ZrIgr8HPDXVAIlP4LHggAZfk0FkSzo8VSXifHCwa2k80g==
dependencies:
cross-fetch "^3.1.5"
cross-fetch "^3.1.6"

mem@^8.1.1:
version "8.1.1"
Expand Down

0 comments on commit 9744fb1

Please sign in to comment.