How To Write Tests

How To Write Tests

Testing style guide for rails grade

Four-phase test

Use clear, four-phase tests.

Tests should be readable in isolation

Students should be able to click "Examine Test" and read and understand a test easily. Therefore, each test needs to be self-contained.

Therefore,

Avoid before, let

Let's avoid mystery guests.

Use a describe block with a target URL and a single descriptive test nested inside

Bad

context "with input 5" do
  it "works", points: 2, hint: h("params_are_strings") do
    visit "/flexible/square/5"

    expect(page).to have_content(25)
  end
end

Good

describe "/flexible/square/5" do
  it "works with input 5", points: 2, hint: h("params_are_strings") do
    visit "/flexible/square/5"

    expect(page).to have_content(25)
  end
end

Avoid deeply nesting describe/feature, context, it/scenario

Bad

feature "Flexible square" do
  describe "/flexible/square/42" do
    it "works with input 5", points: 2, hint: h("params_are_strings") do
      visit "/flexible/square/5"

      expect(page).to have_content(25)
    end
  end

  describe "/flexible/square/42" do
    it "works with input 5", points: 4, hint: h("params_are_strings") do
      visit "/flexible/square/42"

      expect(page).to have_content(1764)
    end
  end
end

Better

describe "/flexible/square/5" do
  it "works with input 5", points: 2, hint: h("params_are_strings") do
    visit "/flexible/square/5"

    expect(page).to have_content(25)
  end

  it "works with input 42", points: 4, hint: h("params_are_strings") do
    visit "/flexible/square/42"

    expect(page).to have_content(1764)
  end
end

Best

describe "/flexible/square/5" do
  it "works with input 5", points: 2, hint: h("params_are_strings") do
    visit "/flexible/square/5"

    expect(page).to have_content(25)
  end
end

describe "/flexible/square/5" do
  it "works with input 42", points: 4, hint: h("params_are_strings") do
    visit "/flexible/square/42"

    expect(page).to have_content(1764)
  end
end

This also allows for tailoring copy more instructively to each individual spec, rather than shoehorning it to fit a DRYer structure.

Test features progressively to break a problem into bite-size steps

Add tests for the presence of hardcoded copy or inputs/labels before testing for behavior. This approach lets students better narrow down what they're missing in their code.

Bad

describe "/photos/new" do
  it "creates a photo when submitted", points: 3, hint: h("button_type") do
    initial_number_of_photos = Photo.count

    visit "/photos/new"

    click_on "Create Photo"

    final_number_of_photos = Photo.count

    expect(final_number_of_photos).to eq(initial_number_of_photos + 1)
  end
end

Good

describe "/photos/new" do
  it "has a form", points: 1 do
    visit "/photos/new"

    expect(page).to have_css("form", count: 1)
  end
end

describe "/photos/new" do
  it "has a label for 'Caption'", points: 1, hint: h("copy_must_match label_for_input") do
    visit "/photos/new"

    expect(page).to have_css("label", text: "Caption")
  end
end

describe "/photos/new" do
  it "has a label for 'Image URL'", points: 1, hint: h("copy_must_match label_for_input") do
    visit "/photos/new"

    expect(page).to have_css("label", text: "Image URL")
  end
end

describe "/photos/new" do
  it "has two inputs", points: 1, hint: h("label_for_input") do
    visit "/photos/new"

    expect(page).to have_css("input", count: 2)
  end
end

describe "/photos/new" do  it "has a button to 'Create Photo'", points: 1, hint: h("copy_must_match") do
    visit "/photos/new"

    expect(page).to have_css("button", text: "Create Photo")
  end
end

describe "/photos/new" do
  it "creates a photo when submitted", points: 3, hint: h("button_type") do
    initial_number_of_photos = Photo.count

    visit "/photos/new"

    click_on "Create Photo"

    final_number_of_photos = Photo.count

    expect(final_number_of_photos).to eq(initial_number_of_photos + 1)
  end
end

One expectation per test

Try to stick to one expectation per test, except when absolutely necessary (e.g. to ensure proper setup).

Ref:

https://devblast.com/b/ruby-testing-with-rspec-one-expectation-per-test

Customized Failure Messages

Use customized failure messages for any expects whose messages are not 100% self-explanatory.

Ref:

https://relishapp.com/rspec/rspec-expectations/docs/customized-message

Hints

You can add hints to each spec to give students just-in-time help even when we aren't standing next time them. This is next-level rubber-ducky stuff.

For example,

it "captures the user's input in the query string with names", points: 4, hint: h("names_for_inputs") do
  visit "/square_root/new"

  expect(page).to have_css("input[name]", count: 1)
end

What's going on above:

• You can provide additional help to the student by optionally including a hint with the it or scenario method. The value should be a string or array of strings.

• Keep hints in I18n so that they don't clutter up the readability of tests, and so that you can easily add the same hint to multiple relevant tests:

    # config/locales/en.yml
  
    en:
      hints:
        names_for_inputs: |
                          Give each `<input>` in your form a unique `name=""` attribute.
  
                          `name=""` is the crucial, functional attribute of an `<input>` that determines what the user's input gets labeled as in the query string, and therefore what key it gets stored under in the `params` hash, and therefore how you will access it in your next RCAV.
  
                          `placeholder=""`, etc, are just helpful attributes to use to be user-friendly. `name=""` is the functional one.

• Store hints under keys under en.hints.

• You can use GitHub-flavored Markdown.

• To further reduce clutter, there is a helper method h() in spec_helper.rb which makes it easy to add multiple hints from I18n:

    def h(hint_identifiers)
      hint_identifiers.split.map { |identifier| I18n.t("hints.#{identifier}") }
    end

  So you can just provide a single string with the I18n keys of multiple hints separated by spaces:

    it "works with 42.42", points: 4, hint: h("label_for_input params_are_strings") do
      visit "/square/new"
  
      fill_in "Enter a number", with: 42.42
  
      click_button "Calculate square"
  
      expect(page).to have_content(1799.4564)
    end

Use factories

I go back and forth over factories vs just ActiveRecord objects, since students know exactly what ActiveRecord objects are.

Currently I lean towards using factories since, used right, they produce minimally valid objects out-of-the-box, and so save so much code.

More important than brevity, however, is that you can then define or re-define only the attributes that are important to the test at hand, thereby drawing attention to them.

I think create(:photo) is intuitive enough for students to guess what it means; it's no more magical to them than the rest of the test code (Capybara methods, etc) that they aren't being explicitly taught.

Capybara selectors

• tried has_content: too much noise

• tried has_css with id: unfamiliar (we could introduce it)

• tried attribute selector, e.g. [data-grade="occurrences"]; too weird looking

• settled on expect(page).to have_css(".occurrences", text: 2) for now since it is very familiar to them, but it feels like class is something that is used for too many other things.

Perhaps teaching id, label, for, and styling with # earlier so that we can select with it might be worthwhile.

When stubbing external requests

Testing external requests is tricky since forcing capybara to wait for a response will have inconsistent results. Therefore, it's best to stub external requests and control the response in our test suite.

Typically, stubbed requests are added in one of the test helper files (either spec/spec_helper.rb or spec/rails_helper.rb ) within an Rspec.configure block.

Stubbed request should be as flexible and forgiving as possible. Use regexp to:

• Allow for http and https

• Allow for mixed case

• Allow for dynamic url segments

• Allow for arbitrary additional query string parameters not specified by us (for example, access tokens)

WIP notes below

http://stackoverflow.com/questions/11377087/can-i-use-capybara-rspec-to-match-a-range

Stubbed requests: Perhaps we create a method for each stubbed request and then call that method at the beginning of the test. Would that be clearer to the student?

---

2022 Notes

HTML/CSS Specs

Prefer Capybara finder methods for checking for HTML elements, content, and attributes since they provide the most clarity and flexibility when writing tests.

The most useful methods are all and find.

Note: find will throw an error if it doesn't find anything. The error message is readable enough for students, but you can still customize it in the expect assertion if you want.

Testing for element

describe "/add" do
  it "has a `<form>` element", points: 1 do
    visit "/add"

    form = find("form")
    expect(form).to_not be_nil,
      "Expected to find one form element on the page but didn't find one."
  end
end

Other ways to write this test:

using rspec-html-matchers

describe "/add" do
  it "has a `<form>` element", points: 1 do
    visit "/add"

    expect(page).to have_tag("form", count: 1),
      "Expected to find one form element on the page but found 0 or more than 1."
  end
end

regular capybara

describe "/add" do
  it "has a `<form>` element", points: 1 do
    visit "/add"

    expect(page).to have_css("form", count: 1),
      "Expected to find one form element on the page but found 0 or more than 1."
  end
end

Testing for an element with specific content

describe "/add" do
  it "has an `<h1>` with the text 'Addition'", points: 1 do
    visit "/add"

    heading = find("h1", :text => /Addition/i)
    expect(heading).to_not be_nil,
      "Expected to find an <h1> with the text 'Addition', but didn't find one."
  end
end

or

describe "/multiply" do
  it "has a label with the text 'by this:'", points: 1, hint: h("copy_must_match label_for_input") do
    visit "/multiply"

    expect(page).to have_css("label", text: /by this/i),
      "Expected to find a 'label' element with specified text, but didn't find one."
  end
end

Testing HTML attributes

You can retrieve values from attributes if you select an element with find or all.

describe "The landing page" do
  it "has a label 'Enter your address below.' with a for attribute that is not empty.", :points => 1 do
    visit "/"
    
    address_label = find("label", :text => /Enter your address below/i)
    for_attribute = address_label[:for]

    expect(for_attribute).to_not be_empty,
      "Expected label's for attribute to be set to a non empty value, was '#{for_attribute}' instead."
  end
end

Use the draft_matchers gem for testing CSS color (by color name), page layout, or element ancestry.

describe "/rock", js: true do
  it "has all elements in the right order", :points => 1 do
    visit "/rock"
    
    play_rock_link = find("a", :text => /Play Rock/i)
    play_paper_link = find("a", :text => /Play Paper/i)
    play_scissors_link = find("a", :text => /Play Scissors/i)

    expect(play_paper_link).to be_below(play_rock_link)

    expect(play_scissors_link).to be_below(play_paper_link)

    rock_heading = find("h2", :text => /We played rock/i)
    
    expect(rock_heading).to be_below(play_scissors_link)
    
    paper_heading = find("h2", :text => /They played paper/i)
    expect(paper_heading).to be_below(rock_heading)
    
    outcome_heading = find("h2", :text => /We lost/i)
    
    expect(outcome_heading).to be_below(paper_heading)
    rules_link = find("a", text: /Rules/i)
    
    expect(rules_link).to be_below(outcome_heading)
  end
end

Note the js: true is required for testing layout.

Testing Copy (in a webpage)

When testing copy and not data that's been created:

• Ignore case

• Ignore punctuation

• Using Regex for this is okay 👌

  • note that extraneous spaces between words should also be okay, since the rendered HTML page will collapse it all down to one space in the end.

Ruby Specs

When writing tests that check for specific output in the Terminal, define this helper in spec_helper.rb, rails_helper.rb, or the spec file.

def with_captured_stdout
  original_stdout = $stdout  # capture previous value of $stdout
  $stdout = StringIO.new     # assign a string buffer to $stdout
  yield                      # perform the body of the user code
  $stdout.string             # return the contents of the string buffer
ensure
  $stdout = original_stdout  # restore $stdout to its previous value
end

Use the method along with the path of the script file that is being tested like this:

output = with_captured_stdout { require_relative('../../string_gets')} 

Use Regex when testing the output:

expect(output.match?(/Hello, Hannah!/)).to be(true)

Make sure the Regex will match the expected output regardless of which printing method a student uses.

You can customize the error message too.

expect(output.match?(/Hello, Hannah!/)).to be(true), "Expected output to be:\nHello, Hannah!\nbut was:\n#{actual_output}"

note: whitespace is respected in the build report in Grades, so add to space out the expected vs actual output on their own lines.

Stubbing method calls

If the script is testing a method that that involves randomness, user input, or time you stub the method call and choose something specific for the test case.

For instance methods

allow_any_instance_of(Object).to receive(:gets).and_return("hannah\n")

For class methods

allow(Date).to receive(:today).and_return Date.new(2020,07,1)

Notes about stubbing gets

Make sure to add to end of the argument to the and_return method since :gets always returns it. If you don't, sometimes weird errors can happen if students use methods we don't expect (like chop instead of chomp).

One full spec might look like this:

  it "should output 'Hello, name!'" , points: 1 do
    allow_any_instance_of(Object).to receive(:gets).and_return("hannah\n")

    output = with_captured_stdout { require_relative('../../string_gets')} 
    output = "empty" if output.empty? 
    expect(output.match?(/Hello, Hannah!/)).to be(true), "Expected output to be:\nHello, Hannah!\nbut was:\n#{output}"
  end

Preventing hardcoded solutions from passing

It would be fairly difficult to prevent all forms of hardcoding for a solution, but preventing the basic "print expected result" is possible by examining the contents of the script file (make sure you ignore commented lines).

script_file = "script.rb"
file_contents = File.read(script_file)
File.foreach(script_file).with_index do |line, line_num|
  if !line.include?("#") || line.include?("p") || line.include?("puts")
    expect(line).to_not match(/5.3/),
      "Expected 'script.rb' to NOT literally print '5.3', but did anyway. On line #{line_num}."
  end
end

Writing multiple tests for the same script

When writing multiple tests for the same script you need to un-require the script in each test first, so that it can be successfully required and the output captured no matter what order the specs are run in. You can un-require a script by using the global Ruby variable $".

# Un-require hash_person.rb
hash_person = $".select{|r| r.include? 'hash_person.rb'}
$".delete(hash_person.first)

Testing that a method has been defined

You can use method_defined? to check that an instance method or attr_accessor has been defined.

describe "class_person.rb" do
  it "has an attribute `birthdate`", points: 1 do
    
    expect(Person.method_defined?(:birthdate)).to eq(true),
      "Expected 'Person' class to have an attr_accessor called 'birthdate', but didn't"
  end
end

Rails Specs

Routes

Testing that a route is visit-able

describe "/rock" do
  it "renders an HTML page", :points => 1 do
    visit "/rock"

    expect(page.status_code).to be(200)
  end
end

if a route should redirect...

describe "/users/[USERNAME]" do
  it "redirects to sign in page when user is signed out", points: 0 do
    user = User.new
    user.password = "password"
    user.username = "user"
    user.email = "[e-mail]"
    user.save

    visit "/users/#{user.username}"
    current_url = page.current_path

    expect(current_url).to eq("/user_sign_in")
  end
end

Forms, params

Write very incremental tests. You should write a test for each of the following:

• form element

  describe "/square/new" do
    it "has one form element", points: 1 do
      visit "/square/new"
  
      expect(page).to have_tag("form", count: 1),
        "Expected to find one form element on the page but found 0 or more than 1."
    end
  end

• form action

  describe "/square/new" do
    it "has form element with an action attribute", points: 1 do
      visit "/square/new"
  
      form = find("form")
      form_action = form[:action]
      expect(form_action).to_not be_nil,
        "Expected form's action attribute to be set to a non empty value, was '#{form_action}' instead."
    end
  end

• label with text

  describe "/square/new" do
    it "has a label with the text 'Enter a number'", points: 1, hint: h("copy_must_match label_for_input") do
      visit "/square/new"
  
      label = find("label", :text => /Enter a number/i)
      expect(label).to_not be_nil,
        "Expected to find a <label> with the text 'Enter a number', but didn't find one."
    end
  end

• count for specific input type (text input, textarea, file input)

  describe "/square_root/new" do
    it "has one input element", points: 1 do
      visit "/square_root/new"
  
      expect(page).to have_tag("input", count: 1)
    end
  end

  you can also use find here with the type attribute.

• button with text

  describe "/square_root/new" do
    it "has a button element with text 'Calculate square root'", points: 1, hint: h("copy_must_match") do
      visit "/square_root/new"
  
      expect(page).to have_tag("button", text: /Calculate square root/i)
    end
  end

• form submits to a real page

  describe "/square_root/new" do
    it "leads to another functional RCAV when submitted", points: 6, hint: h("button_type") do
      visit "/square_root/new"
  
      find("button", :text => /Calculate square root/i ).click
  
      expect(page).to_not have_current_path("/square_root/new", ignore_query: true),
        "Expected form to submit to a different Route, but didn't."
    end
  end

• specific input has name attribute

  describe "/square/new" do
    it "captures the user's input in the query string", points: 1, hint: h("names_for_inputs") do
      visit "/square/new"
  
      expect(page).to have_css("input[name]", count: 1),
        "Expected an input element to have a 'name' attribute but didn't."
    end
  end

• label connected to input

  describe "/square/new" do
    it "has a label that is connected to an input", points: 0, hint: h("label_for_input") do
      visit "/square/new"
  
      number_label = find("label", :text => /Enter a number/i)
      for_attribute = number_label[:for]
  
      if for_attribute.empty?
        expect(for_attribute).to_not be_empty,
          "Expected label’s for attribute to be set to a non empty value, was '#{for_attribute}' instead."
      else
        all_inputs = all("input")
    
        all_input_ids = all_inputs.map { |input| input[:id] }.compact
    
        expect(all_input_ids.count(for_attribute)).to eq(1),
          "Expected label’s for attribute(#{for_attribute}) to match only 1 of the ids of an <input> tag (#{all_input_ids}), but found 0 or more than 1."
      end
    end
  end

• form submits and produces correct content

  describe "/square/new" do
    it "calculates the square correctly with an input of 5", points: 3, hint: h("label_for_input params_are_strings") do
      visit "/square/new"
  
      number_label = find("label", :text => /Enter a number/i)
      for_attribute = number_label[:for]
      begin
        number_input = find("##{for_attribute}")
      rescue Capybara::ElementNotFound
        expect(false). to be(true), "Expected to find an <input> with an id attribute that matched the for attribute of a <label> but didn't find one."
      end
      number_input.set(5)
      find("button", :text => /Calculate square/i ).click
  
      expect(page).to have_content(/25/)
    end
  end

Using find and set for input elements allow for the most flexibility. You can make label text case insensitive and determine what the for and id attributes are set to (if at all) and display them in the error message.

Database / ActiveRecord

Testing that a class/Model has been defined

describe "Movie" do
  it "has a class defined in app/models/", points: 1 do
    expect{ Movie }.to_not raise_error(NameError),
      "Expected a Movie class to be defined in the app/models/ folder, but didn't find one."
  end
end

Testing that a database table has been created

describe "User" do
  it "has an underlying table", points: 0 do
    user_migrations_exists = false
   
    if ActiveRecord::Base.connection.table_exists? "users"
      user_migrations_exists = true
    end
    expect(user_migrations_exists).to be(true),
      "Expected there to be a SQL table called 'users', but didn't find one."
  end
end

Testing that an instance method has been defined

describe "Movie" do
  it "has an instance method defined called 'director'", points: 2 do

    expect(Movie.method_defined?(:director)).to eq(true),
      "Expected Movie class to define an instance method called, 'director', but didn't find one."
  end
end

Testing columns names and data types

  it "has an column called 'password_digest' of type 'string'", points: 0 do
    new_user = User.new
    expect(new_user.attributes).to include("password_digest"),
      "Expected to have a column called 'password_digest', but didn't find one."
    expect(User.column_for_attribute('password_digest').type).to be(:string),
      "Expected column to be of type 'string' but wasn't."
    end

Testing association accessors

Use the shoulda-matchers gem. Ensure the project is setup to work with shoulda-matchers.

# 1-N belongs_to association
RSpec.describe Photo, type: :model do
  describe "has a belongs_to association defined called 'poster' with Class name 'User' and foreign key 'owner_id'", points: 1 do
    it { should belong_to(:poster).class_name("User").with_foreign_key("owner_id") }
  end
end

# 1-N has_many association
RSpec.describe Photo, type: :model do
  describe "has a has_many association defined called 'comments' with Class name 'Comment' and foreign key 'photo_id'", points: 1 do
    it { should have_many(:comments).class_name("Comment").with_foreign_key("photo_id") }
  end
end

# N-N has_many association
RSpec.describe Photo, type: :model do
  describe "has a has_many (many-to_many) association defined called 'fans' through 'likes' and source 'fan'", points: 2 do
    it { should have_many(:fans).through(:likes).source(:fan) }
  end
end

Note: Watch out for the required: true option for 1-N.

Testing with Time

If you're writing tests that relate to time, you can include active support time helpers in rails_helper.rb

RSpec.configure do |config|
  # ...
  config.include ActiveSupport::Testing::TimeHelpers
  # ...
end

Make sure you use Time.current or Date.current instead of Time.now for weird time zone inconsistencies.

Example:

describe "The Done section" do
  it "displays the formatted updated at time for each todo items", points: 1 do
    # ...
    travel_to 12.hours.ago do
      updated_at = Time.current
      within(:css, "div.in_progress") do
        within(:css, "form") do
          find("button", :text => /Move to Done/i ).click
        end
      end
    end

    visit("/")

    formatted_updated_at_time = updated_at.strftime("%-l:%M %p %Z on %A, %d %b %Y")
    expect(page).to have_tag("div.done") do     
      with_tag("ul") do
        with_tag("li", text: /Completed at\s*#{formatted_updated_at_time}/i)
      end
    end
  end
end