require 'json'
require 'rest-client'
require 'nokogiri'

class MozillaSecurityAdvisoryScraper
  attr_accessor :index_url, :default_headers, :advisory_urls

  def initialize()
    @index_url = "https://www.mozilla.org/en-US/security/advisories"
    @default_headers = {
      :accept => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
      :user_agent => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0'
    }
    @advisory_urls ||= get_advisory_urls
  end

  def get_index
    get(index_url)
  end

  def get_advisory(advisory_url)
    get(advisory_url)
  end

  def parse_html(response_body)
    Nokogiri::HTML(response_body)
  end

  def get_and_parse_advisory(advisory_url)
    html = get(advisory_url)
    parse_html(html)
  end

  def advisory_urls_in_html(html_doc)
    html_doc.xpath('//li[@class="level-item"]/a').map {|link| relative_url = link['href']; "https://www.mozilla.org#{relative_url}"}
  end

  def get_advisory_urls
    body = get_index
    doc = parse_html(body)
    advisory_urls_in_html(doc)
  end

  def get(url)
    response = RestClient::Request.execute(
      :method => :get,
      :url => url,
      :headers => default_headers
      )
    if response.code == 200
      response.body
    else
      puts "HTTP Status: #{response.code}"
    end
  end

  def parse_advisory_summary(advisory_doc)
    advisory_doc.xpath('//dl[@class="summary"]').first.css('dd')
  end


  def parse_advisory_cves(advisory_doc)
    advisory_doc.xpath('//section[@class="cve"]')
  end

  def advisory_id_from_url(advisory_url)
    advisory_url.split("/").last
  end

  def data_hash(advisory_doc)
    advisory_url = advisory_doc.xpath('/html/head/meta[7]').last.values.last
    advisory_id = advisory_id_from_url(advisory_url)
    title = advisory_doc.title
    header = advisory_doc.css('h2').text
    summary = parse_advisory_summary(advisory_doc)
    announced = summary[0].text
    impact = summary[1].text
    products = summary[2].text
    fixed_in = summary[3].text.gsub("\n", "").strip
    cve_nodes = parse_advisory_cves(advisory_doc)
    cves = cve_nodes.map { | cve_node | cve_data_hash(cve_node) }

    {
      :advisory_url => advisory_url,
      :advisory_id => advisory_id,
      :title => title,
      :header => header,
      :announced => announced,
      :impact => impact,
      :products => products,
      :fixed_in => fixed_in,
      :cves => cves
    }
  end

  def cve_data_hash(advisory_cve_doc)
    cve_id_and_title = advisory_cve_doc.css('h4').text.gsub("\n", "").gsub("#", "").strip.split(": ")
    cve_id = cve_id_and_title.first 
    cve_title = cve_id_and_title.last
    
    dl_and_dd_tags = advisory_cve_doc.css('dl').css('dd')
    cve_reporter = dl_and_dd_tags.first.text
    cve_impact = dl_and_dd_tags.last.text

    cve_description = advisory_cve_doc.css('p').text
    cve_references = advisory_cve_doc.css('a').last.values.first

    {
      :cve_id => cve_id,
      :title => cve_title,
      :reporter => cve_reporter,
      :impact => cve_impact,
      :description => cve_description,
      :references => cve_references,
    }
 
  end

  def get_all_advisories
    advisory_urls.map do | advisory_url |
      advisory_doc = get_and_parse_advisory(advisory_url)
      data_hash(advisory_doc)
    end
  end

  def advisory_hash_to_json_file(advisory_hash)
    advisory_json = JSON.parse(advisory_hash.to_json)
    pretty_json = JSON.pretty_generate(advisory_json)
    File.write("./data/#{advisory_hash[:advisory_id]}.json", pretty_json)
  end

  def cve_cpes_from_nvd(cve_id)
    body = get("https://services.nvd.nist.gov/rest/json/cpematch/2.0?cveId=#{cve_id}")
    json = JSON.parse(body)
    cpes = json.dig('matchStrings').map {|match_strings| match_strings.dig('matches')}
  end

  def get_mozilla_cpes_from_cve(cve_id)
    cve_cpes = cve_cpes_from_nvd(cve_id)
    cve_cpes.flat_map do |cpe_configuration|
      cpe_configuration.select do |cpe_hash|
        cpe_hash.dig('cpeName').start_with?('cpe:2.3:a:mozilla')
      end
    end.map {|cpe_h| cpe_h.dig('cpeName')}
  end

  def product_name_to_cpe
    # todo: record cpe translation
    {
      'Thunderbird': 'cpe:2.3:a:mozilla:thunderbird',
      'Firefox': 'cpe:2.3:a:mozilla:firefox',
      'Firefox ESR': 'cpe:2.3:a:mozilla:firefox_esr',
      'NSS': 'cpe:2.3:a:mozilla:nss',
      'Mozilla VPN': 'cpe:2.3:a:mozilla:mozilla_vpn',
      'Firefox for Android': '',
      'Firefox for iOS': '',
      'Mozilla VPN Android': '',
      'Mozilla VPN iOS': '',
      'Mozilla VPN Windows': '',
      'Thunderbird ESR': '',
      'SeaMonkey': ''
    }
  end

end
