{ "licence": "CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/), crediting crawlsignals.com. Text quoted from a site belongs to that site's owner and is not under this licence.", "built": "2026-10-03", "records": [ { "host": "api.met.no", "name": "MET Norway API", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "app.verama.com", "name": "Verama", "category": "broker", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "arxiv.org", "name": "arXiv", "category": "reference", "robots": { "read": "2026-10-03", "prohibition": { "text": "Indiscriminate automated downloads from this site are not permitted", "about": [ "automated-access" ] }, "quotes": [] }, "documents": [ { "url": "http://arxiv.org/help/robots", "title": "robots help page", "kind": "policy", "notRead": "robots.txt's comments prohibit crawling, so our crawler reads nothing else from this host", "quotes": [] } ], "access": [], "examples": [] }, { "host": "careers.computas.com", "name": "Computas careers", "category": "job-board", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://careers.computas.com/data-privacy", "title": "Data & Privacy", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://careers.computas.com/privacy-policy", "title": "Privacy policy for the Computas employer branding and recruitment", "kind": "policy", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "catalog.data.gov", "name": "Data.gov Catalog", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.data.gov/privacy-policy", "title": "Privacy Policy", "kind": "policy", "read": "2026-10-03", "quotes": [ { "text": "In most cases, U.S. Federal data available through Data.gov is offered free and without restriction.", "about": [ "licence", "reuse" ] }, { "text": "Non-federal data available through Data.gov may have different licensing.", "about": [ "licence", "reuse" ] } ] } ], "access": [], "examples": [] }, { "host": "cinode.com", "name": "Cinode", "category": "ats", "robots": { "read": "2026-10-03", "quotes": [ { "text": "AI- och sökbotar är medvetet tillåtna. Se llms.txt i roten.", "about": [ "automated-access", "ai-training" ] } ] }, "documents": [ { "url": "https://cinode.com/en/terms/terms-of-service/", "title": "Terms of Service", "kind": "terms", "read": "2026-10-03", "version": "Valid from 15 December 2025.", "law": "This Agreement shall be governed by Swedish law, without reference to the choice and conflict of law provisions thereof.", "quotes": [ { "text": "Provided that the Agreement is complied with, Cinode grants to the Customer and its Affiliated Companies with Authorized Users, a fixed term, worldwide, non-exclusive, terminable, non-transferable, non-sublicensable limited right to access and use the Service during the Agreement, and its Content via the Internet using a supported web browser, solely for Customer’s internal use and solely to perform those functions defined in the Documentation.", "about": [ "reuse", "commercial-use" ] }, { "text": "The use of API is subject to our “Cinode API Terms of Service Agreement”.", "about": [ "api-terms" ] }, { "text": "In addition to the general restrictions in section 5.4. and Cinode’s Acceptable Use Policy, the following shall apply for your use of the AI-assisted Services: Customer must not use the the AI-assisted Services to (a) develop foundation models or other large-scale models that compete with the AI-assisted Services and/or its third party service providers; (b) extract data from the the AI-assisted Services; or to (c) represent that Output was human-generated where it is not", "about": [ "ai-training", "automated-access" ] } ] }, { "url": "https://cinode.com/en/terms/acceptable-use-policy/", "title": "Acceptable Use Policy", "kind": "acceptable-use", "read": "2026-10-03", "version": "LAST UPDATED: 21 November 2025", "quotes": [ { "text": "This means that you are not allowed to help others:", "about": [ "automated-access" ] }, { "id": "aup-robots", "text": "Use robots or other automated means to access and collect data or content from the Service;", "about": [ "automated-access" ] }, { "text": "Neither the Customer nor its Service Users may:", "about": [ "commercial-use", "reuse" ] }, { "text": "Sell, resell, license, relicense, distribute, make available, borrow or rent the Service or in any way exploit the Service for any unauthorized commercial purpose;", "about": [ "commercial-use", "reuse" ] }, { "text": "Frame or mirror any part of the Service or content, other than framing on the Customer’s own intranet or otherwise for its own internal business purposes or as permitted in the documentation;", "about": [ "reuse" ] } ] }, { "url": "https://cinode.com/en/terms/api-terms-of-service/", "title": "Cinode API Terms of Service Agreement", "kind": "api-terms", "read": "2026-10-03", "version": "LAST UPDATED: APR 22 2021", "quotes": [ { "id": "api-license", "text": "These Cinode API Terms grants you a limited, non-exclusive, non-assignable, non-transferable right to use the Cinode API.", "about": [ "api-terms" ] }, { "text": "Scope of acceptable use. You may not use the Cinode API or any other technology in a way that accesses or uses any information beyond what Cinode allows under the API Terms or the Documentation; that changes the Cinode Service; that breaks or bypasses any of Cinode’s technical, administrative, process or security measures; that disrupts or reduces the performance of the Cinode Service or the Cinode API; or that tests the vulnerability of Cinode’s systems or networks.", "about": [ "api-terms", "automated-access" ] }, { "text": "Rate Limits. You will not attempt to exceed or bypass limitations on access, calls and use of the Cinode API, or otherwise use the Cinode API in a manner that exceeds reasonable request volumes, excessive or abusive usage, or otherwise fails to comply with any part of these Cinode API TERMS or the Documentation.", "about": [ "api-terms", "automated-access" ] }, { "text": "Functionality. You may not use the Cinode API to replicate or compete with core services offered by Cinode.", "about": [ "api-terms", "commercial-use" ] }, { "text": "Distribution of your application. You may not distribute or allow access to the Cinode API to anyone other than, if applicable, the company on whose behalf you entered into these Cinode API Terms.", "about": [ "api-terms", "reuse" ] } ] }, { "url": "https://cinode.com/en/api-and-integrations-en/", "title": "API and Integrations", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "id": "rest-api", "text": "Cinode has a well-developed REST API that makes it easy to build integrations with your other tools and systems.", "about": [ "api-terms" ] }, { "id": "mcp-server", "text": "With Cinode MCP Server, you can now connect your own AI agent or LLM (ChatGPT, Claude, etc.) to Cinode – and ask questions directly to your company’s Cinode data.", "about": [ "api-terms", "ai-training" ] } ] }, { "url": "https://cinode.com/integrationer-api/", "title": "API och Integrationer", "kind": "api-docs", "read": "2026-10-03", "quotes": [] }, { "url": "https://cinode.com/en/terms/data-processing-addendum/", "title": "Data Processing Addendum", "kind": "policy", "read": "2026-10-03", "version": "LAST UPDATED: 21 November, 2025", "quotes": [] }, { "url": "https://cinode.com/en/terms/cinode-appmarket-technology-partner-agreement/", "title": "Cinode AppMarket Technology Partner Agreement", "kind": "terms", "read": "2026-10-03", "version": "LAST UPDATE 2022-03-31", "quotes": [] }, { "url": "https://cinode.com/en/privacy/privacy-policy/", "title": "Privacy Policy", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://cinode.com/en/privacy/privacy-policy//", "title": "Privacy Policy", "kind": "policy", "read": "2026-10-03", "quotes": [] } ], "access": [ { "kind": "api", "title": "Cinode REST API", "url": "https://cinode.com/en/api-and-integrations-en/" }, { "kind": "api", "title": "Cinode MCP Server", "url": "https://cinode.com/en/api-and-integrations-en/", "who": "own-data", "whoQuote": "mcp-server" } ], "examples": [] }, { "host": "clef.no", "name": "CLEF", "category": "broker", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://clef.no/personvern", "title": "Personvernerklæring", "kind": "policy", "read": "2026-10-03", "quotes": [ { "text": "Innhold, design og varemerker på nettstedet tilhører CLEF Group AS med mindre annet er oppgitt, og kan ikke gjenbrukes uten tillatelse.", "about": [ "reuse", "permission" ] } ] } ], "access": [], "examples": [] }, { "host": "data.brreg.no", "name": "data.brreg.no", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "data.norge.no", "name": "Felles datakatalog (data.norge.no)", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [ { "text": "Crawl delay (be respectful to your servers)", "about": [ "automated-access" ] } ] }, "documents": [ { "url": "https://data.norge.no/nb/docs/catalogs/data-services", "title": "API-katalogen", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "text": "Gjennom API-katalogen kan det hentes ut maskinlesbare opplysninger om alle adressene (URL-er) som iverksetter et spesifikt API.", "about": [ "api-terms" ] } ] }, { "url": "https://www.digdir.no/digdir/personvernerklaering/706", "title": "Personvernerklæring", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://data.norge.no/specification/dcat-ap-no/", "title": "DCAT-AP-NO", "kind": "other", "read": "2026-10-03", "quotes": [] } ], "access": [ { "kind": "api", "title": "API-katalogen i Felles datakatalog", "url": "https://data.norge.no/nb/docs/catalogs/data-services" } ], "examples": [] }, { "host": "developer.entur.org", "name": "Entur Developer", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://developer.entur.org/stops-and-timetable-data", "title": "Stops- and Timetable data", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "id": "entur-free", "text": "Part of our mission statement is to share our data with anyone who wants it, for free. This page lists all our timetables, and stop place data dumps which are updated daily.", "about": [ "reuse", "licence" ] }, { "id": "entur-licence", "text": "Category: National journey planning. License: NLOD", "about": [ "licence" ] }, { "text": "Excessive downloading of files may lead to your IP being blocked. We expect that most users will have no need for more than one download per 24 hours.", "about": [ "automated-access" ] } ] }, { "url": "https://developer.entur.org/", "title": "Entur developer portal (old)", "kind": "api-docs", "read": "2026-10-03", "quotes": [] }, { "url": "https://developer.entur.no/", "title": "Entur for Developers", "kind": "api-docs", "read": "2026-10-03", "quotes": [] } ], "access": [ { "kind": "bulk", "title": "Stops and timetable data dumps (NeTEx and GTFS)", "url": "https://developer.entur.org/stops-and-timetable-data", "who": "anyone", "whoQuote": "entur-free", "licence": { "id": "NLOD", "quote": "entur-licence" } } ], "examples": [] }, { "host": "en.wikipedia.org", "name": "Wikipedia", "category": "reference", "robots": { "read": "2026-10-03", "quotes": [ { "text": "Please note: There are a lot of pages on this site, and there are some misbehaved spiders out there that go _way_ too fast. If you're irresponsible, your access to the site may be blocked.", "about": [ "automated-access" ] }, { "text": "Some bots are known to be trouble, particularly those designed to copy entire sites. Please obey robots.txt.", "about": [ "automated-access", "reuse" ] }, { "text": "Crawlers that are kind enough to obey, but which we'd rather not have unless they're feeding search engines.", "about": [ "automated-access" ] }, { "text": "Friendly, low-speed bots are welcome viewing article pages, but not dynamically-generated pages please.", "about": [ "automated-access" ] } ] }, "documents": [ { "url": "https://en.wikipedia.org/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_4.0_International_License", "title": "Wikipedia:Text of the Creative Commons Attribution-ShareAlike 4.0 International License", "kind": "licence", "read": "2026-10-03", "quotes": [ { "text": "Under the following terms:", "about": [ "reuse", "licence" ] }, { "text": "Attribution—You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.", "about": [ "reuse", "licence" ] }, { "text": "Share Alike—If you remix, transform, or build upon the material, you must distribute your contributions under the same license as the original.", "about": [ "reuse", "licence" ] }, { "text": "No additional restrictions—You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.", "about": [ "reuse", "licence" ] }, { "text": "Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to:", "about": [ "reuse", "licence" ] }, { "text": "You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material.", "about": [ "reuse", "licence" ] } ] }, { "url": "https://foundation.wikimedia.org/wiki/Special:MyLanguage/Policy:Terms_of_Use", "title": "Terms of Use", "kind": "terms", "notRead": "robots.txt of foundation.wikimedia.org disallows it for our crawler", "quotes": [] }, { "url": "https://foundation.wikimedia.org/wiki/Special:MyLanguage/Legal:Wikimedia_Foundation_Legal_and_Safety_Contact_Information", "title": "Wikimedia Foundation Legal and Safety Contact Information", "kind": "other", "notRead": "robots.txt of foundation.wikimedia.org disallows it for our crawler", "quotes": [] }, { "url": "https://developer.wikimedia.org/", "title": "Wikimedia Developer Portal", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "text": "Access articles from Wikipedia, media files, structured data, and more with public APIs and downloads.", "about": [ "api-terms" ] } ] }, { "url": "https://developer.wikimedia.org/get-started/", "title": "Get started", "kind": "api-docs", "read": "2026-10-03", "quotes": [] }, { "url": "https://foundation.wikimedia.org/wiki/Special:MyLanguage/Terms_of_Use", "title": "Terms of Use", "kind": "terms", "notRead": "robots.txt of foundation.wikimedia.org disallows it for our crawler", "quotes": [] }, { "url": "https://www.gnu.org/licenses/gpl-3.0.html", "title": "GNU General Public License", "kind": "licence", "read": "2026-10-03", "quotes": [ { "text": "All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met.", "about": [ "licence" ] }, { "text": "You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force.", "about": [ "reuse", "licence" ] } ] }, { "url": "https://creativecommons.org/licenses/by-sa/4.0/", "title": "Attribution-ShareAlike 4.0 International", "kind": "licence", "read": "2026-10-03", "quotes": [ { "text": "You are free to:", "about": [ "reuse", "commercial-use", "licence" ] }, { "text": "Share — copy and redistribute the material in any medium or format for any purpose, even commercially.", "about": [ "reuse", "commercial-use", "licence" ] }, { "text": "Adapt — remix, transform, and build upon the material for any purpose, even commercially.", "about": [ "reuse", "commercial-use", "licence" ] }, { "text": "Under the following terms:", "about": [ "reuse", "licence" ] }, { "text": "Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.", "about": [ "reuse", "licence" ] }, { "text": "ShareAlike — If you remix, transform, or build upon the material, you must distribute your contributions under the same license as the original.", "about": [ "reuse", "licence" ] }, { "text": "No additional restrictions — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.", "about": [ "reuse", "licence" ] } ] }, { "url": "https://www.mediawiki.org/wiki/Special:MyLanguage/Developer_Portal", "title": "Developer Portal", "kind": "other", "notRead": "robots.txt of www.mediawiki.org disallows it for our crawler", "quotes": [] }, { "url": "https://www.mediawiki.org/wiki/Special:MyLanguage/Developer_Portal/Contribute", "title": "Developer Portal/Contribute", "kind": "other", "notRead": "robots.txt of www.mediawiki.org disallows it for our crawler", "quotes": [] } ], "access": [ { "kind": "api", "title": "Wikimedia public APIs and downloads", "url": "https://developer.wikimedia.org/" } ], "examples": [] }, { "host": "github.com", "name": "GitHub", "category": "developer", "robots": { "read": "2026-10-03", "quotes": [ { "text": "If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots", "about": [ "automated-access", "permission" ] }, { "text": "We also provide an extensive API: https://docs.github.com", "about": [ "api-terms" ] } ] }, "documents": [ { "url": "https://docs.github.com/integrations/concepts/about-building-integrations", "title": "About building integrations", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "text": "Your integration can use GitHub's API to fetch data and make changes to data on GitHub. GitHub has a REST API and a GraphQL API.", "about": [ "api-terms" ] } ] }, { "url": "https://docs.github.com/site-policy/github-terms/github-terms-of-service", "title": "GitHub Terms of Service", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://docs.github.com/en/rest/about-the-rest-api/comparing-githubs-rest-api-and-graphql-api", "title": "Comparing GitHub's REST API and GraphQL API", "kind": "api-docs", "read": "2026-10-03", "quotes": [] }, { "url": "https://docs.github.com/en/rest", "title": "GitHub REST API documentation", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "id": "rest-intro", "text": "Create integrations, retrieve data, and automate your workflows with the GitHub REST API.", "about": [ "api-terms" ] }, { "text": "You can authenticate to the REST API to access more endpoints and have a higher rate limit.", "about": [ "api-terms" ] } ] }, { "url": "https://docs.github.com/en/graphql", "title": "GitHub GraphQL API documentation", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "id": "graphql-intro", "text": "To create integrations, retrieve data, and automate your workflows, use the GitHub GraphQL API.", "about": [ "api-terms" ] } ] }, { "url": "https://docs.github.com/en/site-policy/github-terms/github-terms-of-service", "title": "GitHub Terms of Service", "kind": "terms", "read": "2026-10-03", "quotes": [ { "id": "tos-bots", "text": "You must be a human to create an Account. Accounts registered by \"bots\" or other automated methods are not permitted.", "about": [ "automated-access" ] }, { "text": "We do permit machine accounts:", "about": [ "automated-access" ] }, { "id": "tos-public-repos", "text": "By choosing to contribute Content to a public repository, you are choosing to and directing us to make such Content accessible to everyone on the internet. Unless specifically set forth herein, these Terms do not restrict lawful access to or use of the contents of public repositories by third parties, or by GitHub or its Affiliates.", "about": [ "reuse", "automated-access" ] }, { "id": "tos-access-reciprocity", "text": "By using automated means to access, collect, or otherwise use (“Access”) any publicly accessible Content from the Service for the purpose of developing or training any commercially available artificial intelligence model, machine learning system, or similar technology (a \"Commercial AI System\"), you hereby waive any and all policies, terms, conditions, or contractual provisions governing products, services, websites or datasets you own or operate that would otherwise prohibit, restrict, or place conditions upon GitHub's Access to any publicly accessible data, information or content associated with your products or services, including for the purpose of developing or training any Commercial AI System. You further agree not to impose technical or other targeted measures to restrict or retaliate against such Access.", "about": [ "automated-access", "ai-training", "commercial-use" ] }, { "text": "This Section D.9 does not apply to Access solely for the purpose of academic research or if, on the date you Access the Content, the number of monthly active users of the products or services made available by you is less than 700 million in the preceding calendar month. For the purposes of this Section, \"you\" shall refer to you and any entity that directly or indirectly controls, is controlled by, or is under common control with you (affiliates).", "about": [ "automated-access", "ai-training" ] }, { "text": "By making a repository public, you grant other Users a nonexclusive, worldwide license to use, display, perform and reproduce (by forking) Your Content through the Service as permitted by GitHub's functionality.", "about": [ "reuse", "licence" ] }, { "text": "You grant GitHub and our Affiliates the right to store, host, archive, parse, display, and make copies of Your Content as necessary to provide, develop, and improve the Service, including by training AI Features, and for the purpose of training, developing, and improving artificial intelligence and machine learning models and technologies of our Affiliates.", "about": [ "ai-training" ] }, { "text": "You may not duplicate, copy, or reuse any portion of the HTML/CSS, JavaScript, or visual design elements or concepts without express written permission from GitHub.", "about": [ "reuse", "permission" ] }, { "text": "This Agreement is licensed under this Creative Commons Zero license.", "about": [ "licence" ] }, { "id": "tos-api-abuse", "text": "Abuse or excessively frequent requests to GitHub via the API may result in the temporary or permanent suspension of your Account's access to the API.", "about": [ "api-terms", "automated-access" ] }, { "text": "You may not share API tokens to exceed GitHub's rate limitations.", "about": [ "api-terms" ] }, { "text": "You may not use the API to download data or Content from GitHub for spamming purposes, including for the purposes of selling GitHub users' personal information, such as to recruiters, headhunters, and job boards.", "about": [ "api-terms", "commercial-use" ] }, { "id": "tos-api-subscription", "text": "GitHub may offer subscription-based access to our API for those Users who require high-throughput access or access that would result in resale of GitHub's Service.", "about": [ "api-terms", "commercial-use" ] } ] }, { "url": "https://docs.github.com/en/site-policy/site-policy-deprecated/amendment-to-github-terms-of-service-applicable-to-us-federal-government-users", "title": "Amendment to GitHub Terms of Service applicable to US Federal Government Users", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://docs.github.com/en/site-policy/github-terms/github-community-guidelines", "title": "GitHub Community Guidelines", "kind": "policy", "read": "2026-10-03", "quotes": [ { "text": "We dedicate these Community Guidelines to the public domain for anyone to use, reuse, adapt, or whatever, under the terms of CC0-1.0.", "about": [ "reuse", "licence" ] } ] }, { "url": "https://docs.github.com/en/site-policy/github-terms/github-terms-for-additional-products-and-features", "title": "GitHub Terms for Additional Products and Features", "kind": "terms", "read": "2026-10-03", "quotes": [ { "text": "The GitHub Advisory Database is licensed under the Creative Commons Attribution 4.0 license. The attribution term may be fulfilled by linking to the GitHub Advisory Database at https://github.com/advisories or to individual GitHub Advisory Database records used, prefixed by https://github.com/advisories.", "about": [ "licence", "reuse", "database-right" ] } ] }, { "url": "https://docs.github.com/en/site-policy/other-site-policies/guidelines-for-legal-requests-of-user-data", "title": "Guidelines for legal requests of user data", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://github.com/github/docs/blob/main/content/site-policy/github-terms/github-terms-of-service.md", "title": "The GitHub Terms of Service", "kind": "other", "read": "2026-10-03", "quotes": [] } ], "access": [ { "kind": "api", "title": "GitHub REST API", "url": "https://docs.github.com/en/rest" }, { "kind": "api", "title": "GitHub GraphQL API", "url": "https://docs.github.com/en/graphql" } ], "examples": [] }, { "host": "huggingface.co", "name": "Hugging Face", "category": "developer", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://huggingface.co/terms-of-service", "title": "Terms of Service", "kind": "terms", "read": "2026-10-03", "version": "Effective Date: September 15, 2022", "law": "the Law of the State of New York", "quotes": [ { "text": "However, by posting your Content or otherwise making it available on our Website, you must be aware that:", "about": [ "reuse", "licence" ] }, { "text": "Your Content will be viewed by others, and therefore:", "about": [ "reuse", "licence" ] }, { "text": "If you decide to set your Repository public, you grant each User a perpetual, irrevocable, worldwide, royalty-free, non-exclusive license to use, display, publish, reproduce, distribute, and make derivative works of your Content through our Services and functionalities;", "about": [ "reuse", "licence" ] }, { "text": "When Content contains notice of a reasonable and customary license, (such as an open source license) such Content is intended to remain under the terms of such license when further accessed, distributed, or used.", "about": [ "reuse", "licence" ] }, { "text": "You may not alter, reproduce, republish, license any of our proprietary materials, unless we expressly give you a written permission to do so.", "about": [ "reuse", "permission" ] } ] }, { "url": "https://huggingface.co/privacy", "title": "Privacy Policy", "kind": "policy", "read": "2026-10-03", "version": "Effective Date: March 28, 2023", "quotes": [] }, { "url": "https://cdn-media.huggingface.co/landing/assets/Supplemental+Terms.pdf", "title": "Supplemental Terms", "kind": "terms", "notRead": "robots.txt of cdn-media.huggingface.co answered status 403, which our fetcher treats as disallow-all", "quotes": [] } ], "access": [], "examples": [] }, { "host": "news.ycombinator.com", "name": "Hacker News", "category": "social", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "pam-stilling-feed.nav.no", "name": "pam-stilling-feed.nav.no", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "portal.emagine.org", "name": "emagine", "category": "broker", "robots": { "read": "2026-10-03", "quotes": [ { "text": "Allow all search engines to index the site", "about": [ "automated-access" ] }, { "text": "Allow crawling of everything except sensitive or duplicate areas", "about": [ "automated-access" ] } ] }, "documents": [], "access": [], "examples": [] }, { "host": "stackoverflow.com", "name": "Stack Overflow", "category": "developer", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "ted.europa.eu", "name": "TED (Tenders Electronic Daily)", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [ { "text": "Allow only default search result page, other urls are dynamic and depends of user inputs", "about": [ "automated-access" ] }, { "text": "Force allowing of simap bulk download page", "about": [ "automated-access" ] } ] }, "documents": [ { "url": "https://ted.europa.eu/en/legal-notice", "title": "Legal notice", "kind": "terms", "read": "2026-10-03", "quotes": [ { "id": "ted-reuse", "text": "Unless otherwise noted, the procurement notices published in the Supplement to the Official Journal of the European Union can be freely reused, for commercial or non-commercial purposes.", "about": [ "reuse", "commercial-use" ] }, { "id": "ted-editorial-cc-by", "text": "The copyright over the editorial content of the SIMAP websites (TED, TED eNotices2, TED Developer Docs and TED Developer Portal) is licensed under the Creative Commons Attribution 4.0 International (CC BY 4.0) license. This means that reuse is allowed provided appropriate credit is given and any changes made are indicated.", "about": [ "reuse", "licence" ] }, { "id": "ted-third-party", "text": "You may be required to clear additional rights if specific content depicts identifiable private individuals or includes third-party works. To use or reproduce content that is not owned by the EU, you may need to seek permission directly from the rightholders.", "about": [ "reuse", "permission" ] }, { "id": "ted-excluded", "text": "Software or documents covered by industrial property rights, such as patents, trademarks, registered designs, logos and names, are excluded from the Commission’s reuse policy and are not licensed to you.", "about": [ "reuse" ] }, { "text": "The SIMAP’s system metadata is dedicated to the public domain in accordance with the Creative Commons Universal Public Domain Dedication deed (CC0 1.0).", "about": [ "reuse", "licence" ] }, { "text": "For all other copyright issues regarding SIMAP, please contact: op-copyright@publications.europa.eu.", "about": [ "permission" ] } ] }, { "url": "https://docs.ted.europa.eu/home/index.html", "title": "TED Developer Docs", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "id": "ted-api", "text": "TED API is a collection of public interfaces allowing 3rd party applications to validate and submit eForms notices, publish tender documents, and search the TED archives.", "about": [ "api-terms" ] }, { "id": "ted-open-data", "text": "Query TED Open Data through our SPARQL endpoint or download TED notice XML files directly.", "about": [ "api-terms", "reuse" ] } ] }, { "url": "https://developer.ted.europa.eu/home", "title": "TED Developer Portal", "kind": "api-docs", "notRead": "the server answered status 200 but the page held no text we could read", "quotes": [] }, { "url": "https://european-union.europa.eu/legal-notice_en", "title": "Legal notice", "kind": "terms", "read": "2026-10-03", "quotes": [ { "text": "Unless otherwise indicated (e.g. in individual copyright notices), content owned by the EU on this website is licensed under the Creative Commons Attribution 4.0 International (CC BY 4.0) licence. This means that reuse is allowed, provided appropriate credit is given and changes are indicated.", "about": [ "reuse", "licence" ] }, { "text": "You may be required to clear additional rights if a specific content depicts identifiable private individuals or includes third-party works. To use or reproduce content that is not owned by the EU, you may need to seek permission directly from the rightholders.", "about": [ "reuse", "permission" ] } ] }, { "url": "https://data.europa.eu/en/legal-notice", "title": "Legal notice", "kind": "terms", "read": "2026-10-03", "quotes": [ { "text": "Unless otherwise noted (e.g. in individual copyright notices), the reuse of the editorial content on this website owned by the EU is authorized under the Creative Commons Attribution 4.0 International (CC BY 4.0) licence. This means that reuse is allowed, provided appropriate credit is given and any changes are indicated.", "about": [ "reuse", "licence" ] }, { "text": "To the extent possible under law, the European Union has waived all copyright and related or neighbouring rights to metadata of the open data portal via Creative Commons CC0 1.0 Universal Public Domain Dedication deed.", "about": [ "reuse", "licence" ] }, { "text": "For the resources without licence information, users must consult the licence conditions in the original portal where the resources were initially published.", "about": [ "reuse", "licence" ] } ] }, { "url": "https://european-union.europa.eu/privacy-policy_en", "title": "Privacy policy for the European Union website", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://european-union.europa.eu/privacy-policy-european-union-website_en", "title": "Privacy policy for the European Union website", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://op.europa.eu/en/web/about-us/legal-notices/helpdesk", "title": "Helpdesk - Privacy statement", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://dataeuropa.gitlab.io/data-provider-manual/legal-notice/cookies/", "title": "Cookies", "kind": "policy", "notRead": "robots.txt of dataeuropa.gitlab.io answered status 403, which our fetcher treats as disallowing everything", "quotes": [] }, { "url": "https://creativecommons.org/licenses/by/4.0/", "title": "Attribution 4.0 International (CC BY 4.0)", "kind": "licence", "read": "2026-10-03", "quotes": [ { "text": "You are free to:", "about": [ "licence", "reuse", "commercial-use" ] }, { "text": "Share — copy and redistribute the material in any medium or format for any purpose, even commercially.", "about": [ "licence", "reuse", "commercial-use" ] }, { "text": "Adapt — remix, transform, and build upon the material for any purpose, even commercially.", "about": [ "licence", "reuse", "commercial-use" ] } ] }, { "url": "http://creativecommons.org/licenses/by/4.0/", "title": "Attribution 4.0 International (CC BY 4.0)", "kind": "licence", "read": "2026-10-03", "quotes": [] } ], "access": [ { "kind": "api", "title": "TED API", "url": "https://docs.ted.europa.eu/home/index.html" }, { "kind": "bulk", "title": "TED Open Data (SPARQL endpoint and TED notice XML files)", "url": "https://docs.ted.europa.eu/home/index.html", "grant": { "quote": "ted-reuse", "limits": [ "ted-third-party", "ted-excluded" ] } } ], "examples": [] }, { "host": "witted.com", "name": "Witted", "category": "broker", "robots": { "read": "2026-10-03", "quotes": [ { "text": "AI / answer engines — explicitly allowed so Witted can be surfaced and cited in AI search (ChatGPT, Perplexity, Google AI Overviews, Claude). Flip any of these to \"Disallow: /\" if Witted decides to opt a given crawler out.", "about": [ "automated-access", "ai-training" ] } ] }, "documents": [ { "url": "https://witted.com/privacy-policy", "title": "Witted Privacy Policy", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://witted.com/privacy-policy/cookies", "title": "Cookies", "kind": "policy", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.airbnb.com", "name": "Airbnb", "category": "marketplace", "robots": { "read": "2026-10-03", "quotes": [ { "text": "If you're a bot who likes crawling webpages, please mind the house rules and avoid accessing any disallowed subfolders to earn a 5-star review from us.", "about": [ "automated-access" ] } ] }, "documents": [], "access": [], "examples": [] }, { "host": "www.amazon.com", "name": "Amazon", "category": "marketplace", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://developer.amazon.com/appsandservices/support/legal/da", "title": "Amazon Developer Services Agreement", "kind": "api-terms", "read": "2026-10-03", "quotes": [ { "text": "You will not take any action related to the Program that interferes with, damages, or accesses or uses in any unauthorized manner the hardware, software, networks, technologies, or other properties or services of ours or of any end user or third party.", "about": [ "api-terms" ] }, { "text": "You may not share your API keys, client credentials, or access tokens (collectively, \"Credentials\") with others and you are responsible for maintaining the security of your Credentials.", "about": [ "api-terms" ] } ] }, { "url": "https://developer.amazon.com/terms-and-agreements", "title": "Amazon Developer Services Agreement", "kind": "api-terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://developer.amazon.com/public/support/legal/tou", "title": "Amazon Developer Services Portal Terms of Use", "kind": "api-terms", "read": "2026-10-03", "quotes": [ { "text": "The Portal is only for your use and you are only authorized to use it in connection with the Developer Program. Amazon grants you a limited, revocable license to access and use the Portal in the course of participating in the Developer Program.", "about": [ "reuse", "commercial-use" ] } ] }, { "url": "https://developer.amazon.com/incentives-api", "title": "Amazon Gift Card API", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "text": "Have questions? Reach out to incentives-api@amazon.com.", "about": [ "permission" ] } ] }, { "url": "https://developer.amazon.com/", "title": "Amazon Developer Services", "kind": "api-docs", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.bbc.com", "name": "BBC", "category": "publisher", "robots": { "read": "2026-10-03", "prohibition": { "text": "No scraping, crawling, or systematic extraction of content", "about": [ "automated-access" ] }, "quotes": [ { "text": "In short: Please use our site like a human, not a robot.", "about": [ "automated-access" ] }, { "text": "No use of BBC content for training or fine-tuning AI models, including large language models (LLMs)", "about": [ "ai-training" ] }, { "text": "No retrieval-augmented generation (RAG), AI-powered search, agentic AI or grounding using BBC content", "about": [ "ai-training" ] }, { "text": "No creating datasets from BBC content", "about": [ "reuse" ] }, { "text": "No text and data mining (TDM) under Article 4 of the EU Directive on Copyright in the Digital Single Market", "about": [ "ai-training" ] }, { "text": "No using BBC content to create summaries for your own use", "about": [ "reuse" ] }, { "text": "No business use without permission (details: https://www.bbc.co.uk/usingthebbc/terms/can-i-use-bbc-content-for-my-business/)", "about": [ "commercial-use", "permission" ] }, { "text": "The BBC reserves all rights in its content and expressly opts out of any statutory exceptions in any jurisdiction for text and data mining, as permitted by law", "about": [ "ai-training", "reuse" ] } ] }, "documents": [ { "url": "https://www.bbc.co.uk/terms", "title": "Terms of Use", "kind": "terms", "notRead": "the robots.txt of www.bbc.co.uk prohibits crawling in its comments, so our crawler reads nothing else there", "quotes": [] }, { "url": "https://www.bbc.co.uk/usingthebbc/terms/can-i-use-bbc-content-for-my-business/", "title": "Can I use BBC content for my business?", "kind": "policy", "notRead": "the robots.txt of www.bbc.co.uk prohibits crawling in its comments, so our crawler reads nothing else there", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.booking.com", "name": "Booking.com", "category": "marketplace", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.brainville.com", "name": "Brainville", "category": "marketplace", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.brainville.com/Content/Legal/Terms%20of%20Use%201_0.pdf", "title": "Terms of Use", "kind": "terms", "read": "2026-10-03", "law": "Swedish law", "quotes": [ { "text": "you agree not to reproduce, duplicate, copy, sell, resell or exploit for any commercial purpose, any portion of the Service, use of the Service or access to the Service;", "about": [ "reuse", "commercial-use" ] }, { "text": "you agree not to interfere with any other user’s right to privacy, including by harvesting or collecting personally-identifiable information about users of the Service or posting private information about a third party.", "about": [ "automated-access", "reuse" ] }, { "text": "Except with our express written permission or as permitted by applicable laws, you may not copy, distribute, reproduce, mirror, frame, publicly display, publicly perform, translate, create derivative works of, re-publish or transmit the Service or Brainville Content (in whole or in part) in any way or through any medium for distribution, publication or any commercial purpose.", "about": [ "reuse", "commercial-use", "permission" ] }, { "text": "You may display, copy and download Brainville Content from the Service solely for your personal and non-commercial use provided that: (a) you do not remove any copyright or proprietary notice from the Brainville Content; (b) such Brainville Content will not be copied or posted on any networked computer or published in any medium; and (c) no modifications are made to such Brainville Content.", "about": [ "reuse", "commercial-use" ] } ] }, { "url": "https://www.brainville.com/Content/Legal/Privacy%20policy%201_0.pdf", "title": "Privacy policy", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.brainville.com/en/API", "title": "Brainville API", "kind": "other", "read": "2026-10-03", "quotes": [] }, { "url": "https://developer.brainville.com/", "title": "Brainville API documentation", "kind": "api-docs", "read": "2026-10-03", "quotes": [] } ], "access": [ { "kind": "api", "title": "Brainville API (assignments, applications, consultant profiles)", "url": "https://developer.brainville.com/" } ], "examples": [] }, { "host": "www.doffin.no", "name": "Doffin", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.ebay.com", "name": "eBay", "category": "marketplace", "robots": { "read": "2026-10-03", "prohibition": { "text": "The use of robots or other automated means to access the eBay site without the express permission of eBay is strictly prohibited.", "about": [ "automated-access" ] }, "quotes": [ { "text": "Notwithstanding the foregoing, eBay may permit automated access to access certain eBay pages but solely for the limited purpose of including content in publicly available search engines.", "about": [ "automated-access", "permission" ] }, { "text": "Any other use of robots or failure to obey the robots exclusion standards set forth at is strictly prohibited.", "about": [ "automated-access" ] }, { "text": "Checkouts are strictly for human users.", "about": [ "automated-access" ] }, { "text": "Automated scraping, buy-for-me agents, LLM-driven bots, or any end-to-end flow that attempts to place orders without human review is strictly prohibited.", "about": [ "automated-access" ] }, { "text": "Unauthorized use of automated agents in checkout may result in legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259", "about": [ "automated-access" ] }, { "text": "Approved enterprise integrations must use our official API and comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement", "about": [ "api-terms", "permission" ] } ] }, "documents": [ { "url": "https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259", "title": "User Agreement", "kind": "terms", "notRead": "robots.txt's comments prohibit crawling, so our crawler reads nothing else from this host", "quotes": [] }, { "url": "https://developer.ebay.com/join/api-license-agreement", "title": "API License Agreement", "kind": "api-terms", "notRead": "the server answered status 403", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.experis.no", "name": "Experis", "category": "job-board", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.facebook.com", "name": "Facebook", "category": "social", "robots": { "read": "2026-10-03", "prohibition": { "text": "Notice: Collection of data on Facebook through automated means is prohibited unless you have express written permission from Facebook and may only be conducted for the limited purpose contained in said permission.", "about": [ "automated-access" ] }, "quotes": [ { "text": "All authorized user-agents listed on this page must comply with Meta’s Automated Data Collection Terms available at: https://www.facebook.com/legal/automated_data_collection_terms", "about": [ "automated-access", "permission" ] } ] }, "documents": [ { "url": "https://www.facebook.com/legal/automated_data_collection_terms", "title": "Automated Data Collection Terms", "kind": "terms", "notRead": "robots.txt's comments prohibit crawling, so our crawler reads nothing else from this host", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.finn.no", "name": "FINN.no", "category": "marketplace", "robots": { "read": "2026-10-03", "prohibition": { "text": "Crawling FINN.no is prohibited unless you have written permission.", "about": [ "automated-access" ] }, "quotes": [ { "text": "Innholdet er beskyttet etter åndsverksloven. Bruk av automatiserte tjenester (roboter, spidere, indeksering m.m.) samt andre fremgangsmåter for systematisk eller regelmessig bruk er ikke tillatt uten eksplisitt samtykke fra FINN.no.", "about": [ "automated-access", "reuse" ] } ] }, "documents": [], "access": [], "examples": [] }, { "host": "www.folq.no", "name": "Folq", "category": "broker", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.folq.no/bruksbetingelser/", "title": "Bruksbetingelser", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.folq.no/personvern/", "title": "Personvernerklæring", "kind": "policy", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.glassdoor.com", "name": "Glassdoor", "category": "job-board", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.google.com", "name": "Google", "category": "reference", "robots": { "read": "2026-10-03", "quotes": [ { "text": "Crawlers of certain social media sites are allowed to access page markup when google.com/imgres* links are shared. To learn more, please contact images-robots-allowlist@google.com.", "about": [ "automated-access", "permission" ] } ] }, "documents": [], "access": [], "examples": [] }, { "host": "www.imdb.com", "name": "IMDb", "category": "reference", "robots": { "read": "2026-10-03", "prohibition": { "text": "Use of any device, tool, or process designed to data mine or scrape the content using automated means is prohibited without prior written permission from IMDb.", "about": [ "automated-access" ] }, "quotes": [ { "text": "The content on this site is owned by IMDb.com, Inc., an Amazon.com company and is made available for your personal, non-commercial use subject to our Conditions of Use here: https://www.imdb.com/conditions/", "about": [ "reuse", "commercial-use" ] }, { "text": "For authorized data access and licensing inquiries, see: https://www.imdb.com/licensing/", "about": [ "permission", "licence" ] } ] }, "documents": [ { "url": "https://www.imdb.com/conditions/", "title": "Conditions of Use", "kind": "terms", "notRead": "robots.txt's comments prohibit crawling, so our crawler reads nothing else from this host", "quotes": [] }, { "url": "https://www.imdb.com/licensing/", "title": "Licensing", "kind": "other", "notRead": "robots.txt's comments prohibit crawling, so our crawler reads nothing else from this host", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.indeed.com", "name": "Indeed", "category": "job-board", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.instagram.com", "name": "Instagram", "category": "social", "robots": { "read": "2026-10-03", "prohibition": { "text": "Notice: Collection of data on Instagram through automated means is prohibited unless you have express written permission from Instagram and may only be conducted for the limited purpose contained in said permission.", "about": [ "automated-access" ] }, "quotes": [ { "text": "All authorized user-agents listed on this page must comply with Meta’s Automated Data Collection Terms available at: https://www.facebook.com/legal/automated_data_collection_terms", "about": [ "automated-access", "permission" ] } ] }, "documents": [ { "url": "https://www.facebook.com/legal/automated_data_collection_terms", "title": "Automated Data Collection Terms", "kind": "terms", "notRead": "the robots.txt of www.facebook.com prohibits crawling in its comments, so our crawler reads nothing else there", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.jobylon.com", "name": "Jobylon", "category": "ats", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.jobylon.com/solutions/data-driven-recruitment", "title": "Data-driven recruitment", "kind": "other", "read": "2026-10-03", "quotes": [ { "text": "Have specific data needs? Use our APIs to connect the data with your own reporting infrastructure to create custom dashboards and reports.", "about": [ "api-terms" ] } ] }, { "url": "https://www.jobylon.com/privacy-policy", "title": "Privacy policy", "kind": "policy", "read": "2026-10-03", "quotes": [] } ], "access": [ { "kind": "api", "title": "Jobylon APIs (reporting data)" } ], "examples": [] }, { "host": "www.kartverket.no", "name": "Kartverket", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.kartverket.no/om-kartverket/personvern", "title": "Personvern", "kind": "policy", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.kons.no", "name": "Kons", "category": "broker", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.kons.no/terms", "title": "Vilkår", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.kons.no/privacy", "title": "Personvern", "kind": "policy", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.linkedin.com", "name": "LinkedIn", "category": "social", "robots": { "read": "2026-10-03", "prohibition": { "text": "The use of robots or other automated means to access LinkedIn without the express permission of LinkedIn is strictly prohibited.", "about": [ "automated-access" ] }, "quotes": [ { "text": "LinkedIn may, in its discretion, permit certain automated access to certain LinkedIn pages, for the limited purpose of including content in approved publicly available search engines.", "about": [ "permission", "automated-access" ] }, { "text": "If you would like to apply for permission to crawl LinkedIn, please email whitelist-crawl@linkedin.com.", "about": [ "permission", "automated-access" ] }, { "text": "Any and all permitted crawling of LinkedIn is subject to LinkedIn's Crawling Terms and Conditions.", "about": [ "permission", "automated-access" ] } ] }, "documents": [ { "url": "https://www.linkedin.com/legal/user-agreement", "title": "User Agreement", "kind": "terms", "notRead": "robots.txt's comments prohibit crawling, so our crawler reads nothing else from this host", "quotes": [] }, { "url": "http://www.linkedin.com/legal/crawling-terms", "title": "Crawling Terms and Conditions", "kind": "terms", "notRead": "robots.txt's comments prohibit crawling, so our crawler reads nothing else from this host", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.ncbi.nlm.nih.gov", "name": "NCBI", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.norway-consulting.no", "name": "Norway Consulting", "category": "broker", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.norway-consulting.no/personvern/", "title": "Personvern", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.norway-consulting.no/it-data-og-analyse/", "title": "Data og Analyse", "kind": "other", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.nytimes.com", "name": "The New York Times", "category": "publisher", "robots": { "read": "2026-10-03", "prohibition": { "text": "Use of any device, tool, or process designed to data mine or scrape the content using automated means is prohibited without prior written permission from The New York Times Company.", "about": [ "automated-access" ] }, "quotes": [ { "text": "New York Times content is made available for your personal, non-commercial use subject to our Terms of Service here: https://help.nytimes.com/hc/en-us/articles/115014893428-Terms-of-Service.", "about": [ "commercial-use", "reuse" ] }, { "text": "Prohibited uses include but are not limited to: (1) text and data mining activities under Art. 4 of the EU Directive on Copyright in the Digital Single Market; (2) the development of any software, machine learning, artificial intelligence (AI), and/or large language models (LLMs); (3) creating or providing archived or cached data sets containing our content to others; and/or (4) any commercial purposes.", "about": [ "ai-training", "commercial-use", "reuse" ] }, { "text": "Contact https://nytlicensing.com/contact/ for assistance.", "about": [ "permission" ] } ] }, "documents": [ { "url": "https://help.nytimes.com/hc/en-us/articles/115014893428-Terms-of-Service", "title": "Terms of Service", "kind": "terms", "read": "2026-10-03", "version": "Effective on January 20, 2026", "quotes": [ { "text": "The contents of the Services, including the Site, are intended for your personal, non-commercial use.", "about": [ "commercial-use", "reuse" ] }, { "text": "Non-commercial use does not include the use of Content without prior written consent from The New York Times Company in connection with: (1) the development of any software program, model, algorithm, or other generative AI tool, including, but not limited to, training or using the Content in connection with the development or operation of a machine learning or artificial intelligence (AI) system (including any use of the Content for training, fine tuning, or grounding the machine learning or AI system or as part of retrieval-augmented generation); or (2) providing archived or cached data sets containing Content to another person or entity.", "about": [ "ai-training", "commercial-use", "reuse" ] }, { "text": "The Services and Content are protected by copyrights, trademarks, patents, trade secrets, database rights, sui generis rights and other intellectual or proprietary rights in or to the Services and Content pursuant to U.S. and international laws.", "about": [ "database-right" ] }, { "text": "You may download or copy certain Content and other downloadable items displayed on the Services for your personal use only, provided that you maintain all copyright and other notices contained in the Content and other downloadable items.", "about": [ "reuse" ] }, { "text": "Copying or storing of any Content for other than personal use is expressly prohibited without prior written permission from The New York Times Rights and Permissions Department, or the copyright holder identified in the copyright notice contained in the Content.", "about": [ "reuse", "permission" ] }, { "text": "Without NYT’s prior written consent, you shall not:", "about": [ "automated-access", "ai-training" ] }, { "text": "use robots, spiders, scripts, service, software or any manual or automatic device, tool, or process designed to data mine or scrape the Content, data or information from the Services, or otherwise use, access, or collect the Content, data or information from the Services using automated means", "about": [ "automated-access" ] }, { "text": "use the Content for the development of any software program, model, algorithm, or generative AI tool, including, but not limited to, training or using the Content in connection with the development or operation of a machine learning or artificial intelligence (AI) system (including any use of the Content for training, fine tuning, or grounding the machine learning or AI system or as part of retrieval-augmented generation).", "about": [ "ai-training" ] }, { "text": "use services, software or any manual or automatic device, tool, or process designed to circumvent any restriction, condition, or technological measure that controls access to the Services in any way, including overriding any security feature, bypassing or circumventing any access controls or use limits of the Services, or failing to abide by exclusionary protocols such as the Robots Exclusion Protocol or the Automated Content Access protocol (NYT’s robots.txt notice does not constitute NYT’s prior written consent under these Terms of Service)", "about": [ "automated-access" ] }, { "text": "cache or archive the Content (except for a public search engine’s use of spiders for creating search indices solely for the inclusion of links and short, non-AI synthesized snippets of the Content in search results)", "about": [ "automated-access", "reuse" ] }, { "text": "perform text and data mining activities under Art. 4 of the EU Directive on Copyright in the Digital Single Market.", "about": [ "ai-training" ] } ] }, { "url": "https://help.nytimes.com/policies/115014893428-Terms-of-Service", "title": "Terms of Service", "kind": "terms", "read": "2026-10-03", "version": "Effective on January 20, 2026", "quotes": [] }, { "url": "https://help.nytimes.com/en_US/115014893428-Terms-of-Service", "title": "Terms of Service", "kind": "terms", "read": "2026-10-03", "version": "Effective on January 20, 2026", "quotes": [] }, { "url": "https://help.nytimes.com/hc/en-us/articles/115014893428", "title": "Terms of Service", "kind": "terms", "read": "2026-10-03", "version": "Effective on January 20, 2026", "quotes": [] }, { "url": "https://help.nytimes.com/fr_FR/115014893428-Terms-of-Service", "title": "Terms of Service (French version)", "kind": "terms", "read": "2026-10-03", "quotes": [ { "text": "Vous ne devez pas effectuer les actions suivantes sans le consentement écrit préalable de NYT:", "about": [ "automated-access" ] }, { "text": "utiliser des robots, des araignées, des scripts, des services, des logiciels ou tout dispositif, outil ou processus manuel ou automatique conçu pour extraire le contenu, les données ou les informations des services, ou autrement utiliser, consulter ou collecter le contenu, les données ou les informations des services à l’aide de moyens automatisés", "about": [ "automated-access" ] } ] }, { "url": "https://help.nytimes.com/pt_BR/115014893428-Terms-of-Service", "title": "Terms of Service (Portuguese version)", "kind": "terms", "read": "2026-10-03", "quotes": [ { "text": "Sem o consentimento prévio por escrito da NYT, você não deve:", "about": [ "automated-access" ] }, { "text": "usar robôs, spiders, scripts, serviços, software ou qualquer dispositivo, ferramenta ou processo manual ou automático projetado para minerar ou extrair dados do Conteúdo, dados ou informações dos Serviços ou, de outra forma, usar, acessar ou coletar o Conteúdo, os dados ou as informações dos Serviços usando meios automatizados", "about": [ "automated-access" ] } ] }, { "url": "https://help.nytimes.com/cn_CN/115014893428-Terms-of-Service", "title": "Terms of Service (Chinese version)", "kind": "terms", "read": "2026-10-03", "quotes": [ { "text": "未经《纽约时报》事先书面同意，用户不得：", "about": [ "automated-access" ] }, { "text": "使用机器人、爬虫软件、脚本、服务、软件或任何手动或自动设备、工具或流程，从服务中进行数据挖掘或获取内容、数据或信息，或使用自动化方式从服务中使用、访问或收集内容、数据或信息", "about": [ "automated-access" ] } ] }, { "url": "https://help.nytimes.com/es_MX/115014893428-Terms-of-Service", "title": "Terms of Service (Spanish version)", "kind": "terms", "read": "2026-10-03", "quotes": [ { "text": "Sin el consentimiento previo por escrito del NYT, usted no deberá:", "about": [ "automated-access" ] }, { "text": "utilizar robots, bots, secuencias de comandos, servicios, software o cualquier dispositivo, herramienta o proceso manual o automático diseñado para buscar y obtener Contenido, datos o información de los Servicios, o recopilar o acceder al Contenido, datos o información de los Servicios gracias a medios automatizados", "about": [ "automated-access" ] } ] }, { "url": "https://help.nytimes.com/policies/10940941449492-The-New-York-Times-Company-Privacy-Policy", "title": "The New York Times Company Privacy Policy", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://help.nytimes.com/hc/en-us/articles/115014893968", "title": "Terms of Sale", "kind": "terms", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.openstreetmap.org", "name": "OpenStreetMap", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [ { "text": "OpenStreetMap's data is available for free in bulk from https://planet.openstreetmap.org", "about": [ "reuse" ] }, { "text": "We encourage you to use these instead of scraping our site.", "about": [ "automated-access" ] }, { "text": "Scraping puts a high load on our donated resources and will lead to your IP being blocked.", "about": [ "automated-access" ] } ] }, "documents": [], "access": [ { "kind": "bulk", "title": "Planet OSM", "url": "https://planet.openstreetmap.org" } ], "examples": [] }, { "host": "www.pinterest.com", "name": "Pinterest", "category": "social", "robots": { "read": "2026-10-03", "quotes": [ { "text": "Allowlist your bot: https://help.pinterest.com/bot-submission-form", "about": [ "automated-access", "permission" ] } ] }, "documents": [], "access": [], "examples": [] }, { "host": "www.reddit.com", "name": "Reddit", "category": "social", "robots": { "read": "2026-10-03", "quotes": [ { "text": "Reddit believes in an open internet, but not the misuse of public content.", "about": [ "reuse" ] }, { "text": "See https://www.reddit.com/r/reddit4researchers/ for details on how Reddit continues to support research and non-commercial use.", "about": [ "permission", "commercial-use" ] } ] }, "documents": [ { "url": "https://support.reddithelp.com/hc/en-us/articles/26410290525844-Public-Content-Policy", "title": "Public Content Policy", "kind": "policy", "notRead": "the server answered status 403 to our crawler", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.reuters.com", "name": "Reuters", "category": "publisher", "robots": { "read": "2026-10-03", "prohibition": { "text": "NOTICE: Collection of content, data and/or information from reuters.com through automated means is prohibited unless you have prior written consent from Reuters and may only be conducted for the purposes explicitly described in such permission.", "about": [ "automated-access" ] }, "quotes": [ { "text": "If you would like to seek permission to license the content and/or other materials of Reuters, please contact https://reutersagency.com/contact-us/enquire/", "about": [ "permission" ] } ] }, "documents": [ { "url": "https://www.reuters.com/info-pages/terms-of-use/", "title": "Terms of Use", "kind": "terms", "notRead": "robots.txt's comments prohibit crawling, so our crawler reads nothing else from this host", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.ryanair.com", "name": "Ryanair", "category": "marketplace", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.sec.gov", "name": "U.S. Securities and Exchange Commission (SEC)", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.sec.gov/about/privacy-information", "title": "SEC Web Site Privacy and Security Policy", "kind": "policy", "read": "2026-10-03", "quotes": [ { "id": "sec-rate", "text": "Current guidelines limit users to a total of no more than 10 requests per second, regardless of the number of machines used to submit requests.", "about": [ "automated-access" ] }, { "id": "sec-bots", "text": "The SEC does not allow \"unclassified\" bots or automated tools to crawl the site.", "about": [ "automated-access" ] }, { "text": "Any request that has been identified as part of an unclassified bot or an automated tool outside of the acceptable policy will be managed to ensure fair access for all users.", "about": [ "automated-access" ] }, { "text": "Information presented on sec.gov is considered public information and may be copied or further distributed by users of the web site without the SEC’s permission.", "about": [ "reuse", "permission" ] } ] }, { "url": "https://www.sec.gov/about/divisions-offices/office-equal-employment-opportunity/no-fear-act-amended", "title": "No Fear Act (as amended)", "kind": "other", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.smartrecruiters.com", "name": "SmartRecruiters", "category": "ats", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.smartrecruiters.com/legal/", "title": "Legal", "kind": "other", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.smartrecruiters.com/legal/terms-of-use/", "title": "Candidate Terms of Use", "kind": "terms", "read": "2026-10-03", "version": "Last modified on 14th May 2019", "law": "the laws of the State of Delaware (USA)", "quotes": [ { "text": "You agree that You will not (please note that this list is not exhaustive):", "about": [ "automated-access" ] }, { "text": "Harvest, collect, gather or assemble information or data regarding other users without their consent;", "about": [ "automated-access" ] }, { "text": "Use automatic means to access content or data from other users;", "about": [ "automated-access" ] } ] }, { "url": "https://www.smartrecruiters.com/legal/terms-and-conditions/", "title": "Customer Terms and conditions", "kind": "terms", "read": "2026-10-03", "version": "Version: November 15, 2024", "quotes": [ { "id": "customer-api-return", "text": "For thirty (30) days after termination or expiration of this Agreement, SmartRecruiters will free of charge, allow Customer to access SmartRecruiters’ Customer Application Programming Interface (“API”) so that Customer can retrieve its data, including, without limitation, Customer Content and Candidate Content, in a format described on https://dev.smartrecruiters.com/customer-api/overview/.", "about": [ "api-terms" ] } ] }, { "url": "https://www.smartrecruiters.com/legal/terms-and-conditions-smartstart/", "title": "SmartStart Customer Terms and Conditions", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.smartrecruiters.com/legal/smartmessage-terms/", "title": "SmartMessage Terms and Conditions", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.smartrecruiters.com/legal/smartjobs-terms/", "title": "SmartJobs Terms and Conditions", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.smartrecruiters.com/legal/attrax-terms/", "title": "Attrax Terms and Conditions", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.smartrecruiters.com/legal/smartpal-terms-and-conditions/", "title": "SmartPal Terms and Conditions", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.smartrecruiters.com/legal/agency-portal-customers-terms-conditions/", "title": "Agency Portal Customers Terms & Conditions", "kind": "terms", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.smartrecruiters.com/legal/general-privacy-policy/", "title": "Privacy policy for website visitors", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.smartrecruiters.com/legal/cookie-policy/", "title": "Cookie Policy", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.smartrecruiters.com/legal/imprint/", "title": "Legal Information", "kind": "other", "read": "2026-10-03", "quotes": [] } ], "access": [ { "kind": "api", "title": "Customer API (data return after termination)", "url": "https://dev.smartrecruiters.com/customer-api/overview/", "who": "own-data", "whoQuote": "customer-api-return" } ], "examples": [] }, { "host": "www.ssb.no", "name": "Statistisk sentralbyrå (SSB)", "category": "public-data", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://www.ssb.no/api", "title": "API", "kind": "api-docs", "read": "2026-10-03", "quotes": [ { "text": "SSB tilbyr API mot Statistikkbanken og mot Klassifikasjoner og kodelister (Klass). Med API kan en raskt hente ut og integrere SSBs data med egne systemer.", "about": [ "api-terms" ] }, { "id": "api-open", "text": "API-ene er åpne og krever ikke registrering.", "about": [ "api-terms" ] }, { "id": "api-licence", "text": "I likhet med resten av ssb.no benytter API-ene lisensen CC BY 4.0.", "about": [ "licence", "api-terms" ] } ] }, { "url": "https://www.ssb.no/diverse/lisens", "title": "Lisens", "kind": "licence", "read": "2026-10-03", "quotes": [ { "id": "site-licence", "text": "Innhold og layout på disse nettsidene er opphavsrettslig beskyttet. Vi benytter lisensen Creative Commons Navngivelse 4.0 Internasjonal (CC BY 4.0).", "about": [ "licence", "reuse" ] }, { "text": "Bearbeide — remixe, endre, og bygge videre på materialet til et hvilket som helst formål, inkludert kommersielle.", "about": [ "reuse", "commercial-use" ] }, { "text": "Data som inneholder personopplysninger og er taushetsbelagt er ikke omfattet av denne lisensen og ikke kan viderebrukes", "about": [ "reuse", "licence" ] } ] }, { "url": "https://creativecommons.org/licenses/by/4.0/deed.no", "title": "Navngivelse 4.0 Internasjonal (CC BY 4.0)", "kind": "licence", "read": "2026-10-03", "quotes": [ { "text": "Du har lov til:", "about": [ "reuse", "commercial-use", "licence" ] }, { "text": "Dele — kopiere, distribuere og spre verket i hvilket som helst medium eller format til et hvilket som helst formål, inkludert kommersielle.", "about": [ "reuse", "commercial-use", "licence" ] }, { "text": "På følgende vilkår:", "about": [ "licence" ] }, { "text": "Navngivelse — Du må oppgi korrekt kreditering, oppgi en lenke til lisensen, og indikere om endringer er blitt gjort. Du kan kan gjøre dette på enhver rimelig måte, men uten at det kan forstås slik at lisensgiver bifaller deg eller din bruk av verket.", "about": [ "licence" ] } ] }, { "url": "https://www.ssb.no/data-til-forskning", "title": "Tilgang til data fra SSB", "kind": "other", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.ssb.no/omssb/personvern", "title": "Personvern og informasjonskapsler", "kind": "policy", "read": "2026-10-03", "quotes": [] } ], "access": [ { "kind": "api", "title": "API for Statistikkbanken (PxWebApi v2) and Klass", "url": "https://www.ssb.no/api", "who": "anyone", "whoQuote": "api-open", "licence": { "id": "CC-BY", "quote": "api-licence" } } ], "examples": [] }, { "host": "www.teamtailor.com", "name": "Teamtailor", "category": "ats", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "https://docs.teamtailor.com/", "title": "Teamtailor API documentation", "kind": "api-docs", "notRead": "the server answered 200 but the page held no text we could read", "quotes": [] }, { "url": "https://www.teamtailor.com/en/privacy-policy/", "title": "Privacy Policy", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://trust.teamtailor.com/", "title": "Trust Center", "kind": "other", "read": "2026-10-03", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.theguardian.com", "name": "The Guardian", "category": "publisher", "robots": { "read": "2026-10-03", "prohibition": { "text": "Any other uses are not permitted, incl. but not limited to: for large language models (LLMs), machine learning and/or artificial intelligence-related purposes; with any of the aforementioned technologies; and/or for any commercial purposes.", "about": [ "reuse", "ai-training", "commercial-use" ] }, "quotes": [ { "text": "Guardian content is made available under our terms and conditions of use.", "about": [ "reuse" ] }, { "text": "Contact licensing@theguardian.com for assistance", "about": [ "permission" ] } ] }, "documents": [ { "url": "https://theguardian.com/license.xml", "title": "license.xml", "kind": "licence", "notRead": "an XML file named in robots.txt; our crawler does not fetch data files", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.tiktok.com", "name": "TikTok", "category": "social", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.tripadvisor.com", "name": "Tripadvisor", "category": "marketplace", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.walmart.com", "name": "Walmart", "category": "marketplace", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [ { "url": "http://walmart.com/help/article/walmart-com-terms-of-use/3b75080af40340d6bbd596f116fae5a0", "title": "Walmart.com Terms of Use", "kind": "terms", "read": "2026-10-03", "version": "Last Updated September 9, 2026", "law": "the applicable laws of the State of Texas", "quotes": [ { "text": "In connection with the Walmart Sites, you will not:", "about": [ "automated-access" ] }, { "text": "Harvest or collect information about users of the Walmart Sites.", "about": [ "automated-access" ] }, { "text": "Systematically download and store any Materials.", "about": [ "automated-access" ] }, { "text": "Use any robot, spider, site search/retrieval application or other manual or automatic device to retrieve, index, “scrape,” “data mine” or otherwise gather any Materials, or use any Materials to develop, train, or improve any artificial intelligence or machine learning model, tool, or technology, or reproduce or circumvent the navigational structure or presentation of the Walmart Sites, without Walmart’s express prior written consent.", "about": [ "automated-access", "ai-training", "permission" ] }, { "text": "Use the Walmart Sites for any commercial purpose, or for any purpose that is fraudulent or otherwise tortious or unlawful.", "about": [ "commercial-use" ] }, { "text": "Reproduce, modify, adapt, translate, create derivative works of, sell, rent, lease, loan, timeshare, distribute, or otherwise exploit any portion of (or any use of) the Walmart Sites except as expressly authorized in these Terms of Use, without Walmart’s express prior written consent.", "about": [ "reuse", "permission" ] }, { "text": "Any collection and use of any product listings, descriptions, or prices; any derivative use of the Walmart Sites or its contents; any downloading, copying, or other use of account information for the benefit of any third party; or any use of data mining, robots, or similar data gathering and extraction tools is strictly prohibited.", "about": [ "automated-access", "reuse" ] }, { "text": "Subject to your compliance with these Terms of Use, and solely for so long as you are permitted by Walmart to use the Walmart Sites, you may access, view, download, and print the Materials for your personal, non-commercial use only; provided, however, that you (1) retain all copyright, trademark, or other proprietary designations contained on all Materials; (2) do not modify or alter the Materials in any way; and (3) do not provide or make available the Materials to any third party in a commercial manner.", "about": [ "reuse", "commercial-use" ] }, { "text": "You may not reproduce (except as noted above), publish, transmit, distribute, display, modify, create derivative works from, sell, or participate in any sale of or exploit in any way, in whole or in part, any of the Materials or the Walmart Sites. The Walmart Sites and Materials may be used only as a personal shopping resource.", "about": [ "reuse" ] }, { "text": "The compilation (meaning the collection, arrangement, and assembly) of the Walmart Sites and Materials is the exclusive property of Walmart and is also protected by U.S. and international copyright laws.", "about": [ "database-right" ] } ] }, { "url": "https://corporate.walmart.com/privacy-security/walmart-privacy-notice", "title": "Walmart Customer Privacy Notice (Online and In-Store)", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.walmart.com/privacyrequest", "title": "Your Privacy Choices", "kind": "policy", "notRead": "the answer held no text we could keep", "quotes": [] }, { "url": "https://corporate.walmart.com/privacy-security/california-privacy-rights", "title": "Walmart California Consumer Privacy Act Notice Summary", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "https://www.walmart.com/help/article/Gift-Card-Terms-and-Conditions/9881a4e10ab24dbe8f406df5c75acefd", "title": "Gift Card Terms and Conditions", "kind": "terms", "read": "2026-10-03", "version": "Last Updated: August 24th 2026", "quotes": [ { "text": "Prohibited activities include but are not limited to the following:", "about": [ "automated-access" ] }, { "text": "Use a bot(s) or similar technology to check Gift Card balances online or through an Interactive Voice Response (IVR).", "about": [ "automated-access" ] } ] }, { "url": "https://www.walmart.com/help/article/international-shipping-terms-of-use/73e7d80c13324db8a8492bafbb7f47f7", "title": "International Shipping Terms of Use", "kind": "terms", "read": "2026-10-03", "version": "Last Updated February 23, 2026", "quotes": [] }, { "url": "https://www.walmart.com/help/article/Third-party-Software-and-Licensing-Notices/a5b3802e3c424fc9ac6872ea93e1c4c5", "title": "Third-party Software and Licensing Notices", "kind": "other", "read": "2026-10-03", "version": "Last Updated November 28, 2023", "quotes": [ { "text": "Without limiting the generality of the foregoing, you agree not to: (a) submit any automated or recorded queries to the Service unless otherwise approved in writing by Walmart and its licensors; (b) use the Service for commercial use; (c) access the Service with software or means other than the Software; (d) copy, reproduce, distribute, or in any other manner duplicate the Software, in whole or in part; (e) sell, lease, license, sublicense, distribute, assign, transfer or otherwise grant any rights in the Software, in whole or in part; (f) modify, port, translate, or create derivative works of the Software; (g) decompile, disassemble, reverse engineer or otherwise attempt to derive, reconstruct, identify or discover any source code, underlying ideas, or algorithms, of the Software or Service by any means; (h) remove any proprietary notices, labels or marks from the Software; (i) use the Service for purposes of comparison with or benchmarking against products or services made available by third-parties; or (j) knowingly take any action that would cause the Software to be placed in the public domain.", "about": [ "automated-access", "commercial-use" ] } ] }, { "url": "https://www.walmart.com/help/article/walmart-terms-of-use/de696dfa1dd4423bb1005668dd19b845", "title": "Walmart+ Terms of Use", "kind": "terms", "read": "2026-10-03", "version": "Last Updated: September 28th, 2026", "quotes": [] }, { "url": "https://www.vizio.com/en/terms/terms-of-service", "title": "VIZIO Product Terms of Service", "kind": "terms", "read": "2026-10-03", "version": "Effective as of: March 16, 2026", "quotes": [ { "text": "Except as explicitly authorized in these Terms of Service, you agree not to archive, download, reproduce, distribute, modify, display, perform, publish, license, create derivative works from, or offer for sale any content contained on or obtained from or through the Platform Services.", "about": [ "reuse" ] } ] } ], "access": [], "examples": [] }, { "host": "www.yelp.com", "name": "Yelp", "category": "marketplace", "robots": { "read": "2026-10-03", "prohibition": { "text": "AI / LLM Crawlers and Agents: Use of any robot, spider, service search/retrieval application, or other automated device, process or means to access, retrieve, copy, scrape, or index any portion of the service or any content is prohibited, except as expressly permitted by Yelp.", "about": [ "automated-access" ] }, "quotes": [ { "text": "If you would like to inquire about crawling Yelp, please contact us at https://www.yelp.com/contact", "about": [ "permission", "automated-access" ] }, { "text": "If you would like to request permission to crawl or use Yelp content, contact us here: https://www.yelp.com/contact", "about": [ "permission" ] } ] }, "documents": [ { "url": "https://www.yelp.com/static?country=US&p=tos", "title": "Terms of Service", "kind": "terms", "notRead": "robots.txt's comments prohibit crawling, so our crawler reads nothing else from this host", "quotes": [] } ], "access": [], "examples": [] }, { "host": "www.youtube.com", "name": "YouTube", "category": "social", "robots": { "read": "2026-10-03", "quotes": [] }, "documents": [], "access": [], "examples": [] }, { "host": "www.zillow.com", "name": "Zillow", "category": "marketplace", "robots": { "read": "2026-10-03", "quotes": [ { "text": "Access to and use of Zillow.com is governed by our Terms of Use. See https://www.zillow.com/corp/Terms.htm", "about": [ "automated-access" ] } ] }, "documents": [ { "url": "https://www.zillow.com/corp/Terms.htm", "title": "Zillow Terms of Use", "kind": "terms", "notRead": "robots.txt disallows it for our crawler", "quotes": [] }, { "url": "https://www.zillow.com/corporate/terms-of-use/", "title": "Zillow Terms of Use", "kind": "terms", "read": "2026-10-03", "version": "Updated October 28, 2025", "quotes": [ { "text": "Prohibited Use. BY USING THE SERVICES, YOU AGREE NOT TO:", "about": [ "automated-access" ] }, { "id": "terms-automated", "text": "conduct automated queries (including screen and database scraping, spiders, robots, crawlers, bypassing “captcha” or similar precautions, or any other automated activity with the purpose of obtaining information from the Services) on the Services;", "about": [ "automated-access" ] }, { "text": "Subject to the restrictions set forth in these Terms of Use, you may copy information from the Services without the aid of any automated processes and only as necessary for your personal use or Pro Use to view, save, print, fax and/or e-mail such information.", "about": [ "automated-access", "reuse" ] }, { "text": "reproduce, modify, distribute, display or otherwise provide access to, create derivative works from, decompile, disassemble, or reverse engineer any portion of the Services, except as explicitly permitted by any Product’s Terms to the extent applicable to that product’s Services;", "about": [ "reuse" ] }, { "text": "reproduce, publicly display, or otherwise make accessible on or through any other website, application, or service any reviews, ratings, or profile information about real estate, lending, or other professionals, underlying images of or information about real estate listings, or other data or content available through the Services, except as explicitly permitted by us for a particular portion of the Services;", "about": [ "reuse" ] }, { "text": "access or use any of the Services to develop competitive products or services; or", "about": [ "commercial-use" ] }, { "text": "Notwithstanding the foregoing, the aggregate level data provided on the Zillow Local-Info Pages (the “Aggregate Data”) may be used for non-personal uses, e.g., real estate market analysis. You may display and distribute derivative works of the Aggregate Data (e.g., within a graph), only so long as the Zillow Companies are cited as a source on every page where the Aggregate Data are displayed, including “Data Provided by Zillow Group.” Such citation may not include any of our logos without our prior written approval or imply any relationship between you and the Zillow Companies beyond that the Zillow Companies are the source of the Aggregate Data.", "about": [ "reuse", "commercial-use", "permission" ] }, { "text": "You are prohibited from displaying any other Zillow Companies’ data without our prior written approval.", "about": [ "reuse", "permission" ] }, { "text": "We do not assert copyright or grant any rights to the underlying images or descriptions of real estate listings provided through the Services.", "about": [ "reuse", "database-right" ] }, { "text": "MLS VOW Terms of Use. In accordance with MLS-specific rules and regulations, you must acknowledge the following before we can display certain real estate listings:", "about": [ "commercial-use", "reuse" ] }, { "text": "All information obtained from Zillow, Inc. will be used solely for your personal, non-commercial purposes.", "about": [ "commercial-use", "reuse" ] }, { "text": "You will not copy, redistribute, or retransmit any of the provided information, except as necessary for considering the purchase or sale of an individual property.", "about": [ "reuse" ] } ] }, { "url": "https://www.zillow.com/z/corp/terms/", "title": "Zillow Terms of Use", "kind": "terms", "read": "2026-10-03", "version": "Updated October 28, 2025", "quotes": [] }, { "url": "https://www.zillow.com/renter-hub/terms/Rental-User-Terms", "title": "Rentals User Terms", "kind": "terms", "read": "2026-10-03", "version": "Updated: June 20, 2025", "quotes": [] }, { "url": "https://www.zillow.com/z/3d-home/guidelines/", "title": "3D Home Guidelines", "kind": "policy", "read": "2026-10-03", "quotes": [] }, { "url": "http://www.zillow.com/mortgage/help/TermsOfUse.htm", "title": "Zillow Group Marketplace, Inc. — Lender Terms of Use", "kind": "terms", "read": "2026-10-03", "version": "Updated October 2025", "quotes": [] }, { "url": "https://www.zillow.com/z/mortgages/terms-of-use/", "title": "Zillow Group Marketplace, Inc. — Lender Terms of Use", "kind": "terms", "read": "2026-10-03", "version": "Updated October 2025", "quotes": [] }, { "url": "https://www.zillow.com/z/mortgages/nmls-data-terms/", "title": "Zillow – NMLS Data Terms", "kind": "terms", "read": "2026-10-03", "version": "Updated November 2020", "quotes": [ { "text": "Permitted Users shall not retain, copy, duplicate, modify, exploit, distribute, transmit, publicly display, reproduce, re-sell, license, edit, prepare derivative works of or incorporate into other works, or translate any NMLS Data.", "about": [ "reuse", "commercial-use" ] }, { "text": "Permitted Users will not reference any NMLS Data in any published materials without first obtaining prior written permission from SRR.", "about": [ "reuse", "permission" ] } ] }, { "url": "http://www.google.com/intl/en_us/help/terms_maps.html", "title": "Google Maps End User Additional Terms of Service", "kind": "terms", "read": "2026-10-03", "version": "Last Modified January 27, 2026", "quotes": [ { "text": "When using Google Maps, you may not (or allow those acting on your behalf to):", "about": [ "automated-access", "reuse" ] }, { "text": "mass download or create bulk feeds of the content (or let anyone else do so);", "about": [ "automated-access", "reuse" ] }, { "text": "redistribute or sell any part of Google Maps or create a new product or service based on Google Maps;", "about": [ "reuse", "commercial-use" ] }, { "text": "use Google Maps to create or augment any other mapping-related dataset (including a mapping or navigation dataset, business listings database, mailing list, or telemarketing list) for use in a service that is a substitute for, or a substantially similar service to, Google Maps; or", "about": [ "reuse", "commercial-use", "database-right" ] } ] }, { "url": "http://www.microsoft.com/maps/assets/docs/terms.aspx", "title": "Microsoft maps terms", "kind": "terms", "notRead": "the address redirected to https://aka.ms/BingMapsEndUserTerms, which our rules do not approve", "quotes": [] } ], "access": [], "examples": [] }, { "host": "x.com", "name": "X", "category": "social", "robots": { "read": "2026-10-03", "quotes": [ { "text": "Wait 1 second between successive requests.", "about": [ "automated-access" ] } ] }, "documents": [], "access": [], "examples": [] } ] }