天天看點
傳回

全文檢索引擎Solr系列——整合MySQL、MongoDBMongodb

MySQL

  1. 拷貝mysql-connector-java-5.1.25-bin.jar到E:\solr-4.8.0\example\solr-webapp\webapp\WEB-INF\lib目錄下面
  2. 配置E:\solr-4.8.0\example\solr\collection1\conf\solrconfig.xml
1 2 3 4 5 6

<

requestHandler

name

=

"/dataimport"

class

=

"org.apache.solr.handler.dataimport.DataImportHandler"

>

<

lst

name

=

"defaults"

>

<

str

name

=

"config"

>data-config.xml</

str

>

</

lst

>

</

requestHandler

>
  1. 導入依賴庫檔案： 
    <pre class="brush: xml; gutter: false; first-line: 1; highlight: []; html-script: false">&lt;lib dir=&quot;../../../dist/&quot; regex=&quot;solr-dataimporthandler-\d.*\.jar&quot;/&gt;</pre>
    加在 
    <pre class="brush: xml; gutter: false; first-line: 1; highlight: []; html-script: false">  &lt;lib dir=&quot;../../../dist/&quot; regex=&quot;solr-cell-\d.*\.jar&quot; /&gt;</pre>
    前面。
  2. 建立E:\solr-4.8.0\example\solr\collection1\conf\data-config.xml，指定MySQL資料庫位址，使用者名、密碼以及建立索引的資料表

    <?

    xml

    version

    =

    "1.0"

    encoding

    =

    "UTF-8"

    ?>

    <

    dataConfig

    > 

    <

    dataSource

    type

    =

    "JdbcDataSource"

    driver

    =

    "com.mysql.jdbc.Driver"

    url

    =

    "jdbc:mysql://localhost:3306/django_blog"

    user

    =

    "root"

    password

    =

    ""

    /> 

    <

    document

    name

    =

    "blog"

    > 

    <

    entity

    name

    =

    "blog_blog"

    pk

    =

    "id"

    query

    =

    "select id,title,content from blog_blog"

    deltaImportQuery

    =

    "select id,title,content from blog_blog where ID='${dataimporter.delta.id}'"

    deltaQuery="select id  from blog_blog where add_time > '${dataimporter.last_index_time}'" 

    deletedPkQuery="select id  from blog_blog where id=0"> 

    <

    field

    column

    =

    "id"

    name

    =

    "id"

    /> 

    <

    field

    column

    =

    "title"

    name

    =

    "title"

    /> 

    <

    field

    column

    =

    "content"

    name

    =

    "content"

    /> 

    </

    entity

    > 

    </

    document

    >

    </

    dataConfig

    >
    • query 用于初次導入到索引的sql語句。
      • 考慮到資料表中的資料量非常大，比如千萬級，不可能一次索引完，是以需要分批次完成，那麼查詢語句query要設定兩個參數：${dataimporter.request.length} ${dataimporter.request.offset}
      • query=”select id,title,content from blog_blog limit ${dataimporter.request.length} offset ${dataimporter.request.offset}”
      • 請求：http://localhost:8983/solr/collection2/dataimport?command=full-import&commit=true&clean=false&offset=0&length=10000
    • deltaImportQuery 根據ID取得需要進入的索引的單條資料。
    • deltaQuery 用于增量索引的sql語句，用于取得需要增量索引的ID。
    • deletedPkQuery 用于取出需要從索引中删除文檔的的ID
  3. 為資料庫表字段建立域（field），編輯E:\solr-4.8.0\example\solr\collection1\conf\schema.xml:

<!-- mysql -->

<

field

name

=

"id"

type

=

"string"

indexed

=

"true"

stored

=

"true"

required

=

"true"

/>

<

field

name

=

"title"

type

=

"text_cn"

indexed

=

"true"

stored

=

"true"

termVectors

=

"true"

termPositions

=

"true"

termOffsets

=

"true"

/>

<

field

name

=

"content"

type

=

"text_cn"

indexed

=

"true"

stored

=

"true"

termVectors

=

"true"

termPositions

=

"true"

termOffsets

=

"true"

/>

<!-- mysql -->
  1. 配置增量索引更新檔案

參考：

  • http://josh-persistence.iteye.com/blog/2017155
  • http://wiki.apache.org/solr/DataImportHandler#Using_delta-import_command

Mongodb

    1. 安裝mongo-connector，最好使用手動安裝方式： 
      git clone https://github.com/10gen-labs/mongo-connector.git
cd mongo-connector
#安裝前修改mongo_connector/constants.py的變量：設定DEFAULT_COMMIT_INTERVAL = 0
python setup.py install
      預設是不會自動送出了，這裡設定成自動送出，否則mongodb資料庫更新，索引這邊沒法同時更新，或者在指令行中可以指定是否自動送出，不過我現在還沒發現。
    2. 配置schema.xml，把mongodb中需要加上索引的字段配置到schema.xml檔案中：

      <?

      xml

      version

      =

      "1.0"

      encoding

      =

      "UTF-8"

      ?>

      <

      schema

      name

      =

      "example"

      version

      =

      "1.5"

      >

      <

      field

      name

      =

      "_version_"

      type

      =

      "long"

      indexed

      =

      "true"

      stored

      =

      "true"

      />

      <

      field

      name

      =

      "_id"

      type

      =

      "string"

      indexed

      =

      "true"

      stored

      =

      "true"

      required

      =

      "true"

      multiValued

      =

      "false"

      />

      <

      field

      name

      =

      "body"

      type

      =

      "string"

      indexed

      =

      "true"

      stored

      =

      "true"

      />

      <

      field

      name

      =

      "title"

      type

      =

      "string"

      indexed

      =

      "true"

      stored

      =

      "true"

      multiValued

      =

      "true"

      />

      <

      field

      name

      =

      "text"

      type

      =

      "text_general"

      indexed

      =

      "true"

      stored

      =

      "false"

      multiValued

      =

      "true"

      />  

      <

      uniqueKey

      >_id</

      uniqueKey

      >

      <

      defaultSearchField

      >title</

      defaultSearchField

      >

      <

      solrQueryParser

      defaultOperator

      =

      "OR"

      />

      <

      fieldType

      name

      =

      "string"

      class

      =

      "solr.StrField"

      sortMissingLast

      =

      "true"

      />

      <

      fieldType

      name

      =

      "long"

      class

      =

      "solr.TrieLongField"

      precisionStep

      =

      "0"

      positionIncrementGap

      =

      "0"

      />

      <

      fieldType

      name

      =

      "text_general"

      class

      =

      "solr.TextField"

      positionIncrementGap

      =

      "100"

      >

      <

      analyzer

      type

      =

      "index"

      >

      <

      tokenizer

      class

      =

      "solr.StandardTokenizerFactory"

      />

      <

      filter

      class

      =

      "solr.StopFilterFactory"

      ignoreCase

      =

      "true"

      words

      =

      "stopwords.txt"

      />

      <

      filter

      class

      =

      "solr.LowerCaseFilterFactory"

      />

      </

      analyzer

      >

      <

      analyzer

      type

      =

      "query"

      >

      <

      tokenizer

      class

      =

      "solr.StandardTokenizerFactory"

      />

      <

      filter

      class

      =

      "solr.StopFilterFactory"

      ignoreCase

      =

      "true"

      words

      =

      "stopwords.txt"

      />

      <

      filter

      class

      =

      "solr.SynonymFilterFactory"

      synonyms

      =

      "synonyms.txt"

      ignoreCase

      =

      "true"

      expand

      =

      "true"

      />

      <

      filter

      class

      =

      "solr.LowerCaseFilterFactory"

      />

      </

      analyzer

      >

      </

      fieldType

      >

      </

      schema

      >
    3. 啟動Mongod： 
      mongod --replSet myDevReplSet --smallfiles
      初始化:rs.initiate()
    4. 啟動mongo-connector: 
      E:\Users\liuzhijun\workspace\mongo-connector\mongo_connector\doc_managers>mongo-connector -m localhost:27017 -t http://localhost:8983/solr/collection2 -n s_soccer.person -u id -d ./solr_doc_manager.py
      • -m：mongod服務
      • -t：solr服務
      • -n：mongodb命名空間，監聽database.collection，多個命名空間逗号分隔
      • -u：uniquekey
      • -d：處理文檔的manager檔案
      注意：mongodb通常使用

      _id

       作為uniquekey，而Solrmore使用

      id

       作為uniquekey，如果不做處理，索引檔案時将會失敗，有兩種方式來處理這個問題：
      1. 指定參數

        --unique-key=id

         到mongo-connector，Mongo Connector 就可以翻譯把

        _id

         轉換到

        id

         。
      2. 把schema.xml檔案中的: 
        <uniqueKey>id<uniqueKey>
        替換成 
        <uniqueKey>_id</uniqueKey>
        同時還要定義一個

        _id

         的字段： 
        <field name="_id" type="string" indexed="true" stored="true" />
      3. 啟動時如果報錯： 
        2014-06-18 12:30:36,648 - ERROR - OplogThread: Last entry no longer in oplog cannot recover! Collection(Database(MongoClient('localhost', 27017), u'local'), u'oplog.rs')
        清空E:\Users\liuzhijun\workspace\mongo-connector\mongo_connector\doc_managers\config.txt中的内容，需要删除索引目錄下的檔案重新啟動

    5. 測試

      mongodb中的資料變化都會同步到solr中去。

轉載于:https://www.cnblogs.com/roam/p/3978479.html

資料庫 Python git
上一篇: VisualBox中linux和windows共享檔案設定
下一篇: 在windows上模拟linux環境,MSYS2——Windows平台下模拟linux環境的搭建-Go語言中文社群...

繼續閱讀