Xapian索引-文档检索过程分析

Xapian::Query term_one = Xapian::Query("T世界");
Xapian::Query term_two = Xapian::Query("T比赛");
Xapian::Query query = Xapian::Query(Xapian::Query::OP_OR, term_one, term_two); // query组装

std::cout "query="  std::endl;

Xapian::Enquire enquire(db);
enquire.set_query(query);
Xapian::MSet result = enquire.get_mset(0, 10); // 执行检索，获取结果
std::cout "find results count="  std::endl;

for (auto it = result.begin(); it != result.end(); ++it) {
    Xapian::Document doc = it.get_document();
    std::string data = doc.get_data();
    double doc_score_weight = it.get_weight();
    int doc_score_percent = it.get_percent();
    std::cout "doc=" ",weight=" ",percent="  std::endl;
}

Query::Query(const string & term, Xapian::termcount wqf, Xapian::termpos pos)
    : internal(new Xapian::Internal::QueryTerm(term, wqf, pos)) {
    LOGCALL_CTOR(API, "Query", term | wqf | pos);
}

Query(op op_, const Xapian::Query & a, const Xapian::Query & b) {
    init(op_, 2);
    bool positional = (op_ == OP_NEAR || op_ == OP_PHRASE);
    add_subquery(positional, a);
    add_subquery(positional, b);
    done();
}

/* 根据OP，生成对应的Query派生类，譬如：or的生成 QueryOr类，含有两个子query，这个QueryOr类对象作为Query的internal成员存在；
在组合多个query时，直接添加到vector中；
如果最后发现vector是空的则将internal设置为NULL，或者=1，则将internal设置为子query的internal，这样子可以避免不必要的vector嵌套，如：[xxquery]，单个元素没必要放在vector中。*/

MSet Enquire::Internal::get_mset(Xapian::doccount first, Xapian::doccount maxitems, Xapian::doccount check_at_least, const RSet *rset, const MatchDecider *mdecider) const {
    LOGCALL(MATCH, MSet, "Enquire::Internal::get_mset", first | maxitems | check_at_least | rset | mdecider);

    if (percent_cutoff && (sort_by == VAL || sort_by == VAL_REL)) {
        throw Xapian::UnimplementedError("Use of a percentage cutoff while sorting primary by value isn‘t currently supported");
    }

    if (weight == 0) {
        weight = new BM25Weight;  // 如果外界没有指定打分策略，采用BM25Weight
    }

    Xapian::doccount first_orig = first;
    {
        Xapian::doccount docs = db.get_doccount();
        first = min(first, docs);
        maxitems = min(maxitems, docs - first);
        check_at_least = min(check_at_least, docs);
        check_at_least = max(check_at_least, first + maxitems);
    }

    AutoPtr<:weight::internal> stats(new Xapian::Weight::Internal);  // 用于记录打分用的全局信息
    // MultiMatch对象的初始化，会执行检索的初始化工作，譬如：填充stats对象，
    ::MultiMatch match(db, query, qlen, rset,
               collapse_max, collapse_key,
               percent_cutoff, weight_cutoff,
               order, sort_key, sort_by, sort_value_forward,
               time_limit, *(stats.get()), weight, spies,
               (sorter.get() != NULL),
               (mdecider != NULL));
    // Run query and put results into supplied Xapian::MSet object.
    MSet retval;
    match.get_mset(first, maxitems, check_at_least, retval, *(stats.get()), mdecider, sorter.get());  // 检索
    if (first_orig != first && retval.internal.get()) {
        retval.internal->firstitem = first_orig;
    }

    Assert(weight->name() != "bool" || retval.get_max_possible() == 0);

    // The Xapian::MSet needs to have a pointer to ourselves, so that it can
    // retrieve the documents.  This is set here explicitly to avoid having
    // to pass it into the matcher, which gets messy particularly in the
    // networked case.
    retval.internal->enquire = this;

    if (!retval.internal->stats) {
        retval.internal->stats = stats.release();
    }

    RETURN(retval);
}

static void prepare_sub_matches(vector > & leaves, Xapian::Weight::Internal & stats) {
    LOGCALL_STATIC_VOID(MATCH, "prepare_sub_matches", leaves | stats);
    // We use a vector to track which SubMatches we‘re already prepared.
    vectorbool> prepared;
    prepared.resize(leaves.size(), false);
    size_t unprepared = leaves.size();
    bool nowait = true;
    while (unprepared) {
        for (size_t leaf = 0; leaf leaf) {
            if (prepared[leaf]) continue;
            SubMatch * submatch = leaves[leaf].get();
            if (!submatch || submatch->prepare_match(nowait, stats)) {
                prepared[leaf] = true;
                --unprepared;
            }
        }
        // Use blocking IO on subsequent passes, so that we don‘t go into
        // a tight loop.
        nowait = false;
    }
}

void MultiMatch::get_mset(Xapian::doccount first, Xapian::doccount maxitems,
             Xapian::doccount check_at_least,
             Xapian::MSet & mset,
             Xapian::Weight::Internal & stats,
             const Xapian::MatchDecider *mdecider,
             const Xapian::KeyMaker *sorter) {
........
}

PostList * LocalSubMatch::get_postlist(MultiMatch * matcher, Xapian::termcount * total_subqs_ptr) {
    LOGCALL(MATCH, PostList *, "LocalSubMatch::get_postlist", matcher | total_subqs_ptr);

    if (query.empty()) {
        RETURN(new EmptyPostList); // MatchNothing
    }

    // Build the postlist tree for the query.  This calls
    // LocalSubMatch::open_post_list() for each term in the query.
    PostList * pl;
    {
        QueryOptimiser opt(*db, *this, matcher);
        pl = query.internal->postlist(&opt, 1.0);
        *total_subqs_ptr = opt.get_total_subqs();
    }

    AutoPtr<:weight> extra_wt(wt_factory->clone());
    // Only uses term-independent stats.
    extra_wt->init_(*stats, qlen);
    if (extra_wt->get_maxextra() != 0.0) {
        // There‘s a term-independent weight contribution, so we combine the
        // postlist tree with an ExtraWeightPostList which adds in this
        // contribution.
        pl = new ExtraWeightPostList(pl, extra_wt.release(), matcher);
    }

    RETURN(pl);
}

LeafPostList * LocalSubMatch::open_post_list(const string& term,
                  Xapian::termcount wqf,
                  double factor,
                  bool need_positions,
                  bool in_synonym,
                  QueryOptimiser * qopt,
                  bool lazy_weight) {
    LOGCALL(MATCH, LeafPostList *, "LocalSubMatch::open_post_list", term | wqf | factor | need_positions | qopt | lazy_weight);

    bool weighted = (factor != 0.0 && !term.empty());

    LeafPostList * pl = NULL;
    if (!term.empty() && !need_positions) {
        if ((!weighted && !in_synonym) ||
            !wt_factory->get_sumpart_needs_wdf_()) {
            Xapian::doccount sub_tf;
            db->get_freqs(term, &sub_tf, NULL);
            if (sub_tf == db->get_doccount()) {
                // If we‘re not going to use the wdf or term positions, and the
                // term indexes all documents, we can replace it with the
                // MatchAll postlist, which is especially efficient if there
                // are no gaps in the docids.
                pl = db->open_post_list(string());
                // Set the term name so the postlist looks up the correct term
                // frequencies - this is necessary if the weighting scheme
                // needs collection frequency or reltermfreq (termfreq would be
                // correct anyway since it‘s just the collection size in this
                // case).
                pl->set_term(term);
            }
        }
    }

    if (!pl) {
        const LeafPostList * hint = qopt->get_hint_postlist();
        if (hint)
            pl = hint->open_nearby_postlist(term);
        if (!pl)
            pl = db->open_post_list(term);
        qopt->set_hint_postlist(pl);
    }

    if (lazy_weight) {
        // Term came from a wildcard, but we may already have that term in the
        // query anyway, so check before accumulating its TermFreqs.
        mapstring, TermFreqs>::iterator i = stats->termfreqs.find(term);
        if (i == stats->termfreqs.end()) {
            Xapian::doccount sub_tf;
            Xapian::termcount sub_cf;
            db->get_freqs(term, &sub_tf, &sub_cf);
            stats->termfreqs.insert(make_pair(term, TermFreqs(sub_tf, 0, sub_cf)));
        }
    }

    if (weighted) {
        Xapian::Weight * wt = wt_factory->clone();
        if (!lazy_weight) {
            wt->init_(*stats, qlen, term, wqf, factor);  // BM25Weight::init()计算不涉及query跟doc相关性部分的打分（只跟term和query相关）
            stats->set_max_part(term, wt->get_maxpart());
        } else {
            // Delay initialising the actual weight object, so that we can
            // gather stats for the terms lazily expanded from a wildcard
            // (needed for the remote database case).
            wt = new LazyWeight(pl, wt, stats, qlen, wqf, factor);
        }
        pl->set_termweight(wt);
    }
    RETURN(pl);
}

PostList * MultiAndPostList::find_next_match(double w_min) {
advanced_plist0:
    if (plist[0]->at_end()) {
        did = 0;
        return NULL;
    }
    did = plist[0]->get_docid();
    for (size_t i = 1; i i) {
        bool valid;
        check_helper(i, did, w_min, valid);
        if (!valid) {
            next_helper(0, w_min);
            goto advanced_plist0;
        }
        if (plist[i]->at_end()) {
            did = 0;
            return NULL;
        }
        Xapian::docid new_did = plist[i]->get_docid();
        if (new_did != did) {
            skip_to_helper(0, new_did, w_min);
            goto advanced_plist0;
        }
    }
    return NULL;
}

double LeafPostList::get_weight() const {
    if (!weight) return 0;
    Xapian::termcount doclen = 0, unique_terms = 0;
    // Fetching the document length and number of unique terms is work we can
    // avoid if the weighting scheme doesn‘t use them.
    if (need_doclength)
        doclen = get_doclength();
    if (need_unique_terms)
        unique_terms = get_unique_terms();
    double sumpart = weight->get_sumpart(get_wdf(), doclen, unique_terms);
    AssertRel(sumpart, get_maxpart());
    return sumpart;
}